2024-03-07T02:36:32.529 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-03-07T02:36:32.529 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi --description upgrade:reef-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --owner scheduled_teuthology@teuthology --verbose --archive /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583725 -- /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583725/orig.config.yaml 2024-03-07T02:36:32.588 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-07T02:36:32.688 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583725 branch: main description: upgrade:reef-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' first_in_suite: false job_id: '7583725' kernel: kdb: true sha1: distro ktype: distro last_in_suite: false machine_type: smithi meta: - desc: 'librbd C and C++ api tests ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 workunit: branch: main sha1: ac30ee6cd9cb289983b4db31a5b462d33a1efd30 owner: scheduled_teuthology@teuthology priority: 50 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 1775 sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: reef clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:reef-x suite_branch: main suite_path: /home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: ac30ee6cd9cb289983b4db31a5b462d33a1efd30 targets: smithi145.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMAL6Vof3GB7HXmCHAMjxiFOSDAwUGkVVUw2QFv3Yrl6zMKqu7qDxT/v4ir4M0Zv9hvHfDpa/xMU3HykgvyQNfU= smithi189.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB5LvkjG6SOg20+l0hQzF1QR+7a+iTehPmeeXt6RZI2ajhMOpI1dHaL3E8yUeZ6sEIwh31Aqph0XFwy+7W5Di/w= tasks: - install: branch: reef exclude_packages: - ceph-volume - cephadm: compiled_cephadm_branch: reef conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:reef - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-03-07_02:25:02 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-03-07T02:36:32.688 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa; will attempt to use it 2024-03-07T02:36:32.689 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa/tasks 2024-03-07T02:36:32.689 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-03-07T02:36:32.691 INFO:teuthology.task.internal:Checking packages... 2024-03-07T02:36:32.714 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'ea9a5688d9980e4ddedb59679d2960bcbac6fc71' 2024-03-07T02:36:32.714 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-07T02:36:32.714 INFO:teuthology.packaging:ref: None 2024-03-07T02:36:32.714 INFO:teuthology.packaging:tag: None 2024-03-07T02:36:32.714 INFO:teuthology.packaging:branch: main 2024-03-07T02:36:32.714 INFO:teuthology.packaging:sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T02:36:32.715 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=main 2024-03-07T02:36:33.088 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-1905-gea9a5688-1jammy 2024-03-07T02:36:33.089 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-03-07T02:36:33.096 INFO:teuthology.task.internal:no buildpackages task found 2024-03-07T02:36:33.096 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-03-07T02:36:33.136 INFO:teuthology.task.internal:Saving configuration 2024-03-07T02:36:33.157 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-03-07T02:36:33.163 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-07T02:36:33.189 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi145.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583725', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-07 02:26:06.344561', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF/SNaMGYNdsMhjWvPrwEnY8d9RiD151l04j6gTxjTWr'} 2024-03-07T02:36:33.212 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi189.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583725', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-07 02:26:06.343249', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPeFWazZudVBuekJ5OIZkVvHPoOlFqA0aITrr1auz56j'} 2024-03-07T02:36:33.212 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-03-07T02:36:33.219 INFO:teuthology.task.internal:roles: ubuntu@smithi145.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-03-07T02:36:33.219 INFO:teuthology.task.internal:roles: ubuntu@smithi189.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-03-07T02:36:33.219 INFO:teuthology.run_tasks:Running task console_log... 2024-03-07T02:36:33.310 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f3f1dbe1160>, signals=[15]) 2024-03-07T02:36:33.310 INFO:teuthology.run_tasks:Running task internal.connect... 2024-03-07T02:36:33.316 INFO:teuthology.task.internal:Opening connections... 2024-03-07T02:36:33.316 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi145.front.sepia.ceph.com 2024-03-07T02:36:33.318 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:36:33.399 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi189.front.sepia.ceph.com 2024-03-07T02:36:33.400 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:36:33.481 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-03-07T02:36:33.488 DEBUG:teuthology.orchestra.run.smithi145:> uname -m 2024-03-07T02:36:33.493 INFO:teuthology.orchestra.run.smithi145.stdout:x86_64 2024-03-07T02:36:33.493 DEBUG:teuthology.orchestra.run.smithi145:> cat /etc/os-release 2024-03-07T02:36:33.539 INFO:teuthology.orchestra.run.smithi145.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-03-07T02:36:33.540 INFO:teuthology.orchestra.run.smithi145.stdout:NAME="Ubuntu" 2024-03-07T02:36:33.540 INFO:teuthology.orchestra.run.smithi145.stdout:VERSION_ID="22.04" 2024-03-07T02:36:33.540 INFO:teuthology.orchestra.run.smithi145.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-03-07T02:36:33.540 INFO:teuthology.orchestra.run.smithi145.stdout:VERSION_CODENAME=jammy 2024-03-07T02:36:33.540 INFO:teuthology.orchestra.run.smithi145.stdout:ID=ubuntu 2024-03-07T02:36:33.540 INFO:teuthology.orchestra.run.smithi145.stdout:ID_LIKE=debian 2024-03-07T02:36:33.540 INFO:teuthology.orchestra.run.smithi145.stdout:HOME_URL="https://www.ubuntu.com/" 2024-03-07T02:36:33.540 INFO:teuthology.orchestra.run.smithi145.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-03-07T02:36:33.540 INFO:teuthology.orchestra.run.smithi145.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-03-07T02:36:33.540 INFO:teuthology.orchestra.run.smithi145.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-03-07T02:36:33.540 INFO:teuthology.orchestra.run.smithi145.stdout:UBUNTU_CODENAME=jammy 2024-03-07T02:36:33.542 INFO:teuthology.lock.ops:Updating smithi145.front.sepia.ceph.com on lock server 2024-03-07T02:36:33.562 DEBUG:teuthology.orchestra.run.smithi189:> uname -m 2024-03-07T02:36:33.566 INFO:teuthology.orchestra.run.smithi189.stdout:x86_64 2024-03-07T02:36:33.567 DEBUG:teuthology.orchestra.run.smithi189:> cat /etc/os-release 2024-03-07T02:36:33.612 INFO:teuthology.orchestra.run.smithi189.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-03-07T02:36:33.612 INFO:teuthology.orchestra.run.smithi189.stdout:NAME="Ubuntu" 2024-03-07T02:36:33.613 INFO:teuthology.orchestra.run.smithi189.stdout:VERSION_ID="22.04" 2024-03-07T02:36:33.613 INFO:teuthology.orchestra.run.smithi189.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-03-07T02:36:33.613 INFO:teuthology.orchestra.run.smithi189.stdout:VERSION_CODENAME=jammy 2024-03-07T02:36:33.613 INFO:teuthology.orchestra.run.smithi189.stdout:ID=ubuntu 2024-03-07T02:36:33.613 INFO:teuthology.orchestra.run.smithi189.stdout:ID_LIKE=debian 2024-03-07T02:36:33.613 INFO:teuthology.orchestra.run.smithi189.stdout:HOME_URL="https://www.ubuntu.com/" 2024-03-07T02:36:33.613 INFO:teuthology.orchestra.run.smithi189.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-03-07T02:36:33.613 INFO:teuthology.orchestra.run.smithi189.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-03-07T02:36:33.613 INFO:teuthology.orchestra.run.smithi189.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-03-07T02:36:33.613 INFO:teuthology.orchestra.run.smithi189.stdout:UBUNTU_CODENAME=jammy 2024-03-07T02:36:33.613 INFO:teuthology.lock.ops:Updating smithi189.front.sepia.ceph.com on lock server 2024-03-07T02:36:33.628 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-03-07T02:36:33.638 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-03-07T02:36:33.660 INFO:teuthology.task.internal:Checking for old test directory... 2024-03-07T02:36:33.661 DEBUG:teuthology.orchestra.run.smithi145:> test '!' -e /home/ubuntu/cephtest 2024-03-07T02:36:33.663 DEBUG:teuthology.orchestra.run.smithi189:> test '!' -e /home/ubuntu/cephtest 2024-03-07T02:36:33.668 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-03-07T02:36:33.686 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-03-07T02:36:33.686 DEBUG:teuthology.orchestra.run.smithi145:> test -z $(ls -A /var/lib/ceph) 2024-03-07T02:36:33.709 DEBUG:teuthology.orchestra.run.smithi189:> test -z $(ls -A /var/lib/ceph) 2024-03-07T02:36:33.796 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-03-07T02:36:33.855 INFO:teuthology.run_tasks:Running task kernel... 2024-03-07T02:36:33.871 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-03-07T02:36:33.872 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-03-07T02:36:33.872 DEBUG:teuthology.orchestra.run.smithi145:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-07T02:36:33.872 DEBUG:teuthology.orchestra.run.smithi189:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-07T02:36:33.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-07T02:36:33.877 DEBUG:teuthology.orchestra.run.smithi145:> uname -r 2024-03-07T02:36:33.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-07T02:36:33.878 DEBUG:teuthology.orchestra.run.smithi189:> uname -r 2024-03-07T02:36:33.924 INFO:teuthology.orchestra.run.smithi145.stdout:5.15.0-56-generic 2024-03-07T02:36:33.924 INFO:teuthology.task.kernel:Running kernel on smithi145: 5.15.0-56-generic 2024-03-07T02:36:33.924 DEBUG:teuthology.orchestra.run.smithi145:> sudo apt-get clean 2024-03-07T02:36:33.925 INFO:teuthology.orchestra.run.smithi189.stdout:5.15.0-56-generic 2024-03-07T02:36:33.925 INFO:teuthology.task.kernel:Running kernel on smithi189: 5.15.0-56-generic 2024-03-07T02:36:33.926 DEBUG:teuthology.orchestra.run.smithi189:> sudo apt-get clean 2024-03-07T02:36:34.061 DEBUG:teuthology.orchestra.run.smithi145:> sudo apt-get update 2024-03-07T02:36:34.092 DEBUG:teuthology.orchestra.run.smithi189:> sudo apt-get update 2024-03-07T02:36:34.197 INFO:teuthology.orchestra.run.smithi145.stdout:Get:1 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB] 2024-03-07T02:36:34.240 INFO:teuthology.orchestra.run.smithi189.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-03-07T02:36:34.250 INFO:teuthology.orchestra.run.smithi189.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB] 2024-03-07T02:36:34.250 INFO:teuthology.orchestra.run.smithi189.stdout:Get:3 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB] 2024-03-07T02:36:34.299 INFO:teuthology.orchestra.run.smithi145.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-03-07T02:36:34.335 INFO:teuthology.orchestra.run.smithi189.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [109 kB] 2024-03-07T02:36:34.381 INFO:teuthology.orchestra.run.smithi145.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB] 2024-03-07T02:36:34.417 INFO:teuthology.orchestra.run.smithi145.stdout:Get:4 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [415 kB] 2024-03-07T02:36:34.505 INFO:teuthology.orchestra.run.smithi189.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,217 kB] 2024-03-07T02:36:34.583 INFO:teuthology.orchestra.run.smithi189.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,433 kB] 2024-03-07T02:36:34.593 INFO:teuthology.orchestra.run.smithi145.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,217 kB] 2024-03-07T02:36:34.725 INFO:teuthology.orchestra.run.smithi145.stdout:Get:6 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [220 kB] 2024-03-07T02:36:34.730 INFO:teuthology.orchestra.run.smithi145.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [109 kB] 2024-03-07T02:36:34.738 INFO:teuthology.orchestra.run.smithi145.stdout:Get:8 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [11.4 kB] 2024-03-07T02:36:34.738 INFO:teuthology.orchestra.run.smithi145.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [1,495 kB] 2024-03-07T02:36:34.777 INFO:teuthology.orchestra.run.smithi145.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [33.7 kB] 2024-03-07T02:36:34.777 INFO:teuthology.orchestra.run.smithi145.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [248 kB] 2024-03-07T02:36:34.782 INFO:teuthology.orchestra.run.smithi145.stdout:Get:12 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [520 B] 2024-03-07T02:36:34.782 INFO:teuthology.orchestra.run.smithi145.stdout:Get:13 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [596 kB] 2024-03-07T02:36:34.800 INFO:teuthology.orchestra.run.smithi189.stdout:Get:7 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [415 kB] 2024-03-07T02:36:34.800 INFO:teuthology.orchestra.run.smithi189.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [581 kB] 2024-03-07T02:36:34.816 INFO:teuthology.orchestra.run.smithi189.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [280 kB] 2024-03-07T02:36:34.819 INFO:teuthology.orchestra.run.smithi145.stdout:Get:14 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [847 kB] 2024-03-07T02:36:34.825 INFO:teuthology.orchestra.run.smithi145.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [161 kB] 2024-03-07T02:36:34.825 INFO:teuthology.orchestra.run.smithi189.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [16.1 kB] 2024-03-07T02:36:34.826 INFO:teuthology.orchestra.run.smithi145.stdout:Get:16 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [16.8 kB] 2024-03-07T02:36:34.826 INFO:teuthology.orchestra.run.smithi145.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.1 kB] 2024-03-07T02:36:34.827 INFO:teuthology.orchestra.run.smithi189.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [35.3 kB] 2024-03-07T02:36:34.827 INFO:teuthology.orchestra.run.smithi145.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-03-07T02:36:34.827 INFO:teuthology.orchestra.run.smithi145.stdout:Get:19 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,476 B] 2024-03-07T02:36:34.827 INFO:teuthology.orchestra.run.smithi145.stdout:Get:20 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [260 B] 2024-03-07T02:36:34.828 INFO:teuthology.orchestra.run.smithi189.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [1,523 kB] 2024-03-07T02:36:34.851 INFO:teuthology.orchestra.run.smithi189.stdout:Get:13 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [220 kB] 2024-03-07T02:36:34.863 INFO:teuthology.orchestra.run.smithi189.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [253 kB] 2024-03-07T02:36:34.868 INFO:teuthology.orchestra.run.smithi189.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [520 B] 2024-03-07T02:36:34.868 INFO:teuthology.orchestra.run.smithi189.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,053 kB] 2024-03-07T02:36:34.869 INFO:teuthology.orchestra.run.smithi145.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [581 kB] 2024-03-07T02:36:34.877 INFO:teuthology.orchestra.run.smithi189.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [11.4 kB] 2024-03-07T02:36:34.878 INFO:teuthology.orchestra.run.smithi189.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [33.7 kB] 2024-03-07T02:36:34.907 INFO:teuthology.orchestra.run.smithi189.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [692 kB] 2024-03-07T02:36:34.907 INFO:teuthology.orchestra.run.smithi189.stdout:Get:20 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [1,495 kB] 2024-03-07T02:36:34.914 INFO:teuthology.orchestra.run.smithi189.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [237 kB] 2024-03-07T02:36:34.917 INFO:teuthology.orchestra.run.smithi189.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [22.1 kB] 2024-03-07T02:36:34.917 INFO:teuthology.orchestra.run.smithi189.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,160 B] 2024-03-07T02:36:34.918 INFO:teuthology.orchestra.run.smithi189.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [42.1 kB] 2024-03-07T02:36:34.918 INFO:teuthology.orchestra.run.smithi189.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.1 kB] 2024-03-07T02:36:34.919 INFO:teuthology.orchestra.run.smithi189.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [472 B] 2024-03-07T02:36:34.923 INFO:teuthology.orchestra.run.smithi189.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [41.7 kB] 2024-03-07T02:36:34.924 INFO:teuthology.orchestra.run.smithi189.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [33.9 kB] 2024-03-07T02:36:34.940 INFO:teuthology.orchestra.run.smithi189.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [10.5 kB] 2024-03-07T02:36:34.940 INFO:teuthology.orchestra.run.smithi189.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-03-07T02:36:34.940 INFO:teuthology.orchestra.run.smithi189.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [13.4 kB] 2024-03-07T02:36:34.943 INFO:teuthology.orchestra.run.smithi189.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [24.3 kB] 2024-03-07T02:36:34.943 INFO:teuthology.orchestra.run.smithi189.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-03-07T02:36:34.943 INFO:teuthology.orchestra.run.smithi189.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [644 B] 2024-03-07T02:36:35.051 INFO:teuthology.orchestra.run.smithi189.stdout:Get:35 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [248 kB] 2024-03-07T02:36:35.063 INFO:teuthology.orchestra.run.smithi145.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,433 kB] 2024-03-07T02:36:35.070 INFO:teuthology.orchestra.run.smithi189.stdout:Get:36 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [520 B] 2024-03-07T02:36:35.070 INFO:teuthology.orchestra.run.smithi189.stdout:Get:37 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [847 kB] 2024-03-07T02:36:35.122 INFO:teuthology.orchestra.run.smithi189.stdout:Get:38 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [596 kB] 2024-03-07T02:36:35.150 INFO:teuthology.orchestra.run.smithi189.stdout:Get:39 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [161 kB] 2024-03-07T02:36:35.157 INFO:teuthology.orchestra.run.smithi189.stdout:Get:40 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [16.8 kB] 2024-03-07T02:36:35.158 INFO:teuthology.orchestra.run.smithi189.stdout:Get:41 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-03-07T02:36:35.158 INFO:teuthology.orchestra.run.smithi189.stdout:Get:42 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.1 kB] 2024-03-07T02:36:35.160 INFO:teuthology.orchestra.run.smithi189.stdout:Get:43 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,476 B] 2024-03-07T02:36:35.160 INFO:teuthology.orchestra.run.smithi189.stdout:Get:44 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [260 B] 2024-03-07T02:36:35.207 INFO:teuthology.orchestra.run.smithi145.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [280 kB] 2024-03-07T02:36:35.223 INFO:teuthology.orchestra.run.smithi145.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [16.1 kB] 2024-03-07T02:36:35.223 INFO:teuthology.orchestra.run.smithi145.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [1,523 kB] 2024-03-07T02:36:35.282 INFO:teuthology.orchestra.run.smithi145.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [35.3 kB] 2024-03-07T02:36:35.283 INFO:teuthology.orchestra.run.smithi145.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [253 kB] 2024-03-07T02:36:35.290 INFO:teuthology.orchestra.run.smithi145.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [520 B] 2024-03-07T02:36:35.454 INFO:teuthology.orchestra.run.smithi145.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [692 kB] 2024-03-07T02:36:36.036 INFO:teuthology.orchestra.run.smithi145.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,053 kB] 2024-03-07T02:36:36.148 INFO:teuthology.orchestra.run.smithi145.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [237 kB] 2024-03-07T02:36:36.167 INFO:teuthology.orchestra.run.smithi145.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [22.1 kB] 2024-03-07T02:36:36.168 INFO:teuthology.orchestra.run.smithi145.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,160 B] 2024-03-07T02:36:36.169 INFO:teuthology.orchestra.run.smithi145.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [42.1 kB] 2024-03-07T02:36:36.172 INFO:teuthology.orchestra.run.smithi145.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.1 kB] 2024-03-07T02:36:36.173 INFO:teuthology.orchestra.run.smithi145.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [472 B] 2024-03-07T02:36:36.173 INFO:teuthology.orchestra.run.smithi145.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [41.7 kB] 2024-03-07T02:36:36.176 INFO:teuthology.orchestra.run.smithi145.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [33.9 kB] 2024-03-07T02:36:36.178 INFO:teuthology.orchestra.run.smithi145.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [10.5 kB] 2024-03-07T02:36:36.343 INFO:teuthology.orchestra.run.smithi145.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-03-07T02:36:36.426 INFO:teuthology.orchestra.run.smithi145.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [13.4 kB] 2024-03-07T02:36:36.507 INFO:teuthology.orchestra.run.smithi145.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [24.3 kB] 2024-03-07T02:36:36.556 INFO:teuthology.orchestra.run.smithi145.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-03-07T02:36:36.615 INFO:teuthology.orchestra.run.smithi145.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [644 B] 2024-03-07T02:36:40.305 INFO:teuthology.orchestra.run.smithi189.stdout:Fetched 12.0 MB in 2s (6,221 kB/s) 2024-03-07T02:36:40.832 INFO:teuthology.orchestra.run.smithi145.stdout:Fetched 12.0 MB in 2s (4,794 kB/s) 2024-03-07T02:36:41.655 INFO:teuthology.orchestra.run.smithi189.stdout:Reading package lists... 2024-03-07T02:36:41.673 DEBUG:teuthology.orchestra.run.smithi189:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-03-07T02:36:41.725 INFO:teuthology.orchestra.run.smithi189.stdout:Reading package lists... 2024-03-07T02:36:41.897 INFO:teuthology.orchestra.run.smithi189.stdout:Building dependency tree... 2024-03-07T02:36:41.898 INFO:teuthology.orchestra.run.smithi189.stdout:Reading state information... 2024-03-07T02:36:42.046 INFO:teuthology.orchestra.run.smithi189.stdout:The following packages were automatically installed and are no longer required: 2024-03-07T02:36:42.046 INFO:teuthology.orchestra.run.smithi189.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-03-07T02:36:42.046 INFO:teuthology.orchestra.run.smithi189.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-03-07T02:36:42.047 INFO:teuthology.orchestra.run.smithi189.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-03-07T02:36:42.047 INFO:teuthology.orchestra.run.smithi189.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-03-07T02:36:42.047 INFO:teuthology.orchestra.run.smithi189.stdout: python2.7-minimal 2024-03-07T02:36:42.047 INFO:teuthology.orchestra.run.smithi189.stdout:Use 'sudo apt autoremove' to remove them. 2024-03-07T02:36:42.048 INFO:teuthology.orchestra.run.smithi189.stdout:The following additional packages will be installed: 2024-03-07T02:36:42.048 INFO:teuthology.orchestra.run.smithi189.stdout: linux-generic linux-headers-5.15.0-97 linux-headers-5.15.0-97-generic 2024-03-07T02:36:42.048 INFO:teuthology.orchestra.run.smithi189.stdout: linux-headers-generic linux-image-5.15.0-97-generic 2024-03-07T02:36:42.048 INFO:teuthology.orchestra.run.smithi189.stdout: linux-modules-5.15.0-97-generic linux-modules-extra-5.15.0-97-generic 2024-03-07T02:36:42.049 INFO:teuthology.orchestra.run.smithi189.stdout:Suggested packages: 2024-03-07T02:36:42.049 INFO:teuthology.orchestra.run.smithi189.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-03-07T02:36:42.050 INFO:teuthology.orchestra.run.smithi189.stdout:Recommended packages: 2024-03-07T02:36:42.050 INFO:teuthology.orchestra.run.smithi189.stdout: thermald 2024-03-07T02:36:42.088 INFO:teuthology.orchestra.run.smithi189.stdout:The following NEW packages will be installed: 2024-03-07T02:36:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: linux-headers-5.15.0-97 linux-headers-5.15.0-97-generic 2024-03-07T02:36:42.088 INFO:teuthology.orchestra.run.smithi189.stdout: linux-image-5.15.0-97-generic linux-modules-5.15.0-97-generic 2024-03-07T02:36:42.089 INFO:teuthology.orchestra.run.smithi189.stdout: linux-modules-extra-5.15.0-97-generic 2024-03-07T02:36:42.090 INFO:teuthology.orchestra.run.smithi189.stdout:The following packages will be upgraded: 2024-03-07T02:36:42.090 INFO:teuthology.orchestra.run.smithi189.stdout: linux-generic linux-headers-generic linux-image-generic 2024-03-07T02:36:42.157 INFO:teuthology.orchestra.run.smithi189.stdout:3 upgraded, 5 newly installed, 0 to remove and 288 not upgraded. 2024-03-07T02:36:42.157 INFO:teuthology.orchestra.run.smithi189.stdout:Need to get 113 MB of archives. 2024-03-07T02:36:42.157 INFO:teuthology.orchestra.run.smithi189.stdout:After this operation, 582 MB of additional disk space will be used. 2024-03-07T02:36:42.157 INFO:teuthology.orchestra.run.smithi189.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-97-generic amd64 5.15.0-97.107 [22.6 MB] 2024-03-07T02:36:42.158 INFO:teuthology.orchestra.run.smithi145.stdout:Reading package lists... 2024-03-07T02:36:42.174 DEBUG:teuthology.orchestra.run.smithi145:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-03-07T02:36:42.224 INFO:teuthology.orchestra.run.smithi145.stdout:Reading package lists... 2024-03-07T02:36:42.396 INFO:teuthology.orchestra.run.smithi145.stdout:Building dependency tree... 2024-03-07T02:36:42.397 INFO:teuthology.orchestra.run.smithi145.stdout:Reading state information... 2024-03-07T02:36:42.546 INFO:teuthology.orchestra.run.smithi145.stdout:The following packages were automatically installed and are no longer required: 2024-03-07T02:36:42.547 INFO:teuthology.orchestra.run.smithi145.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-03-07T02:36:42.547 INFO:teuthology.orchestra.run.smithi145.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-03-07T02:36:42.547 INFO:teuthology.orchestra.run.smithi145.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-03-07T02:36:42.547 INFO:teuthology.orchestra.run.smithi145.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-03-07T02:36:42.547 INFO:teuthology.orchestra.run.smithi145.stdout: python2.7-minimal 2024-03-07T02:36:42.547 INFO:teuthology.orchestra.run.smithi145.stdout:Use 'sudo apt autoremove' to remove them. 2024-03-07T02:36:42.548 INFO:teuthology.orchestra.run.smithi145.stdout:The following additional packages will be installed: 2024-03-07T02:36:42.548 INFO:teuthology.orchestra.run.smithi145.stdout: linux-generic linux-headers-5.15.0-97 linux-headers-5.15.0-97-generic 2024-03-07T02:36:42.548 INFO:teuthology.orchestra.run.smithi145.stdout: linux-headers-generic linux-image-5.15.0-97-generic 2024-03-07T02:36:42.549 INFO:teuthology.orchestra.run.smithi145.stdout: linux-modules-5.15.0-97-generic linux-modules-extra-5.15.0-97-generic 2024-03-07T02:36:42.550 INFO:teuthology.orchestra.run.smithi145.stdout:Suggested packages: 2024-03-07T02:36:42.550 INFO:teuthology.orchestra.run.smithi145.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-03-07T02:36:42.550 INFO:teuthology.orchestra.run.smithi145.stdout:Recommended packages: 2024-03-07T02:36:42.550 INFO:teuthology.orchestra.run.smithi145.stdout: thermald 2024-03-07T02:36:42.590 INFO:teuthology.orchestra.run.smithi145.stdout:The following NEW packages will be installed: 2024-03-07T02:36:42.590 INFO:teuthology.orchestra.run.smithi145.stdout: linux-headers-5.15.0-97 linux-headers-5.15.0-97-generic 2024-03-07T02:36:42.590 INFO:teuthology.orchestra.run.smithi145.stdout: linux-image-5.15.0-97-generic linux-modules-5.15.0-97-generic 2024-03-07T02:36:42.591 INFO:teuthology.orchestra.run.smithi145.stdout: linux-modules-extra-5.15.0-97-generic 2024-03-07T02:36:42.591 INFO:teuthology.orchestra.run.smithi145.stdout:The following packages will be upgraded: 2024-03-07T02:36:42.592 INFO:teuthology.orchestra.run.smithi145.stdout: linux-generic linux-headers-generic linux-image-generic 2024-03-07T02:36:42.770 INFO:teuthology.orchestra.run.smithi145.stdout:3 upgraded, 5 newly installed, 0 to remove and 288 not upgraded. 2024-03-07T02:36:42.770 INFO:teuthology.orchestra.run.smithi145.stdout:Need to get 113 MB of archives. 2024-03-07T02:36:42.771 INFO:teuthology.orchestra.run.smithi145.stdout:After this operation, 582 MB of additional disk space will be used. 2024-03-07T02:36:42.771 INFO:teuthology.orchestra.run.smithi145.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-97-generic amd64 5.15.0-97.107 [22.6 MB] 2024-03-07T02:36:42.792 INFO:teuthology.orchestra.run.smithi189.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-97-generic amd64 5.15.0-97.107 [11.5 MB] 2024-03-07T02:36:42.973 INFO:teuthology.orchestra.run.smithi189.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-97-generic amd64 5.15.0-97.107 [63.8 MB] 2024-03-07T02:36:44.191 INFO:teuthology.orchestra.run.smithi145.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-97-generic amd64 5.15.0-97.107 [11.5 MB] 2024-03-07T02:36:44.593 INFO:teuthology.orchestra.run.smithi145.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-97-generic amd64 5.15.0-97.107 [63.8 MB] 2024-03-07T02:36:44.695 INFO:teuthology.orchestra.run.smithi189.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.97.92 [1,696 B] 2024-03-07T02:36:44.696 INFO:teuthology.orchestra.run.smithi189.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.97.92 [2,494 B] 2024-03-07T02:36:44.788 INFO:teuthology.orchestra.run.smithi189.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-97 all 5.15.0-97.107 [12.3 MB] 2024-03-07T02:36:45.124 INFO:teuthology.orchestra.run.smithi189.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-97-generic amd64 5.15.0-97.107 [2,892 kB] 2024-03-07T02:36:45.207 INFO:teuthology.orchestra.run.smithi189.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.97.92 [2,342 B] 2024-03-07T02:36:45.701 INFO:teuthology.orchestra.run.smithi189.stdout:Fetched 113 MB in 3s (36.3 MB/s) 2024-03-07T02:36:45.887 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package linux-modules-5.15.0-97-generic. 2024-03-07T02:36:46.887 INFO:teuthology.orchestra.run.smithi145.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.97.92 [1,696 B] 2024-03-07T02:36:46.887 INFO:teuthology.orchestra.run.smithi145.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.97.92 [2,494 B] 2024-03-07T02:36:46.888 INFO:teuthology.orchestra.run.smithi145.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-97 all 5.15.0-97.107 [12.3 MB] 2024-03-07T02:36:47.691 INFO:teuthology.orchestra.run.smithi145.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-97-generic amd64 5.15.0-97.107 [2,892 kB] 2024-03-07T02:36:47.820 INFO:teuthology.orchestra.run.smithi145.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.97.92 [2,342 B] 2024-03-07T02:36:47.866 INFO:teuthology.orchestra.run.smithi189.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-03-07T02:36:47.871 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../0-linux-modules-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:36:47.951 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking linux-modules-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:36:48.211 INFO:teuthology.orchestra.run.smithi145.stdout:Fetched 113 MB in 5s (21.7 MB/s) 2024-03-07T02:36:48.373 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package linux-modules-5.15.0-97-generic. 2024-03-07T02:36:50.085 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package linux-image-5.15.0-97-generic. 2024-03-07T02:36:50.106 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../1-linux-image-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:36:50.198 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking linux-image-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:36:50.375 INFO:teuthology.orchestra.run.smithi145.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-03-07T02:36:50.379 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../0-linux-modules-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:36:50.446 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking linux-modules-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:36:50.772 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-97-generic. 2024-03-07T02:36:50.792 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:36:51.005 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking linux-modules-extra-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:36:52.540 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package linux-image-5.15.0-97-generic. 2024-03-07T02:36:52.561 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../1-linux-image-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:36:52.661 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking linux-image-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:36:53.117 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-97-generic. 2024-03-07T02:36:53.138 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:36:53.167 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking linux-modules-extra-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:36:57.285 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../3-linux-generic_5.15.0.97.92_amd64.deb ... 2024-03-07T02:36:57.392 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking linux-generic (5.15.0.97.92) over (5.15.0.56.54) ... 2024-03-07T02:36:57.717 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.97.92_amd64.deb ... 2024-03-07T02:36:57.838 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking linux-image-generic (5.15.0.97.92) over (5.15.0.56.54) ... 2024-03-07T02:36:58.132 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package linux-headers-5.15.0-97. 2024-03-07T02:36:58.155 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../5-linux-headers-5.15.0-97_5.15.0-97.107_all.deb ... 2024-03-07T02:36:58.199 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking linux-headers-5.15.0-97 (5.15.0-97.107) ... 2024-03-07T02:36:59.781 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../3-linux-generic_5.15.0.97.92_amd64.deb ... 2024-03-07T02:36:59.906 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking linux-generic (5.15.0.97.92) over (5.15.0.56.54) ... 2024-03-07T02:37:00.196 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.97.92_amd64.deb ... 2024-03-07T02:37:00.301 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking linux-image-generic (5.15.0.97.92) over (5.15.0.56.54) ... 2024-03-07T02:37:00.596 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package linux-headers-5.15.0-97. 2024-03-07T02:37:00.618 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../5-linux-headers-5.15.0-97_5.15.0-97.107_all.deb ... 2024-03-07T02:37:00.662 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking linux-headers-5.15.0-97 (5.15.0-97.107) ... 2024-03-07T02:37:03.902 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package linux-headers-5.15.0-97-generic. 2024-03-07T02:37:03.939 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../6-linux-headers-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:37:03.977 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking linux-headers-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:37:06.409 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package linux-headers-5.15.0-97-generic. 2024-03-07T02:37:06.446 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../6-linux-headers-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:37:06.484 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking linux-headers-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:37:06.558 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.97.92_amd64.deb ... 2024-03-07T02:37:06.681 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking linux-headers-generic (5.15.0.97.92) over (5.15.0.56.54) ... 2024-03-07T02:37:07.022 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up linux-headers-5.15.0-97 (5.15.0-97.107) ... 2024-03-07T02:37:07.144 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up linux-headers-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:37:07.270 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up linux-headers-generic (5.15.0.97.92) ... 2024-03-07T02:37:07.382 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up linux-image-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:37:08.716 INFO:teuthology.orchestra.run.smithi189.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-03-07T02:37:08.717 INFO:teuthology.orchestra.run.smithi189.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-03-07T02:37:08.717 INFO:teuthology.orchestra.run.smithi189.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-97-generic 2024-03-07T02:37:08.717 INFO:teuthology.orchestra.run.smithi189.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-97-generic 2024-03-07T02:37:08.875 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up linux-modules-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:37:09.041 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.97.92_amd64.deb ... 2024-03-07T02:37:09.165 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking linux-headers-generic (5.15.0.97.92) over (5.15.0.56.54) ... 2024-03-07T02:37:09.490 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up linux-headers-5.15.0-97 (5.15.0-97.107) ... 2024-03-07T02:37:09.586 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up linux-headers-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:37:09.704 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up linux-headers-generic (5.15.0.97.92) ... 2024-03-07T02:37:10.505 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up linux-image-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:37:10.568 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up linux-modules-extra-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:37:11.830 INFO:teuthology.orchestra.run.smithi145.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-03-07T02:37:11.830 INFO:teuthology.orchestra.run.smithi145.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-03-07T02:37:11.830 INFO:teuthology.orchestra.run.smithi145.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-97-generic 2024-03-07T02:37:11.830 INFO:teuthology.orchestra.run.smithi145.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-97-generic 2024-03-07T02:37:11.944 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up linux-image-generic (5.15.0.97.92) ... 2024-03-07T02:37:11.985 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up linux-modules-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:37:12.070 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up linux-generic (5.15.0.97.92) ... 2024-03-07T02:37:12.180 INFO:teuthology.orchestra.run.smithi189.stdout:Processing triggers for linux-image-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:37:12.234 INFO:teuthology.orchestra.run.smithi189.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-03-07T02:37:12.234 INFO:teuthology.orchestra.run.smithi189.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-97-generic 2024-03-07T02:37:13.411 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up linux-modules-extra-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:37:14.788 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up linux-image-generic (5.15.0.97.92) ... 2024-03-07T02:37:14.914 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up linux-generic (5.15.0.97.92) ... 2024-03-07T02:37:15.040 INFO:teuthology.orchestra.run.smithi145.stdout:Processing triggers for linux-image-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:37:15.102 INFO:teuthology.orchestra.run.smithi145.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-03-07T02:37:15.103 INFO:teuthology.orchestra.run.smithi145.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-97-generic 2024-03-07T02:37:26.467 INFO:teuthology.orchestra.run.smithi189.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-03-07T02:37:26.467 INFO:teuthology.orchestra.run.smithi189.stdout:Sourcing file `/etc/default/grub' 2024-03-07T02:37:26.476 INFO:teuthology.orchestra.run.smithi189.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-03-07T02:37:26.491 INFO:teuthology.orchestra.run.smithi189.stdout:Generating grub configuration file ... 2024-03-07T02:37:26.736 INFO:teuthology.orchestra.run.smithi189.stdout:Found linux image: /boot/vmlinuz-5.15.0-97-generic 2024-03-07T02:37:26.759 INFO:teuthology.orchestra.run.smithi189.stdout:Found initrd image: /boot/initrd.img-5.15.0-97-generic 2024-03-07T02:37:27.011 INFO:teuthology.orchestra.run.smithi189.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-03-07T02:37:27.017 INFO:teuthology.orchestra.run.smithi189.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-03-07T02:37:27.081 INFO:teuthology.orchestra.run.smithi189.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-03-07T02:37:27.087 INFO:teuthology.orchestra.run.smithi189.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-03-07T02:37:27.233 INFO:teuthology.orchestra.run.smithi189.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-03-07T02:37:27.233 INFO:teuthology.orchestra.run.smithi189.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-03-07T02:37:27.233 INFO:teuthology.orchestra.run.smithi189.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-03-07T02:37:27.281 INFO:teuthology.orchestra.run.smithi189.stdout:done 2024-03-07T02:37:27.778 DEBUG:teuthology.orchestra.run.smithi189:> dpkg -s linux-image-generic 2024-03-07T02:37:27.803 INFO:teuthology.orchestra.run.smithi189.stdout:Package: linux-image-generic 2024-03-07T02:37:27.803 INFO:teuthology.orchestra.run.smithi189.stdout:Status: install ok installed 2024-03-07T02:37:27.803 INFO:teuthology.orchestra.run.smithi189.stdout:Priority: optional 2024-03-07T02:37:27.804 INFO:teuthology.orchestra.run.smithi189.stdout:Section: kernel 2024-03-07T02:37:27.804 INFO:teuthology.orchestra.run.smithi189.stdout:Installed-Size: 21 2024-03-07T02:37:27.804 INFO:teuthology.orchestra.run.smithi189.stdout:Maintainer: Ubuntu Kernel Team 2024-03-07T02:37:27.804 INFO:teuthology.orchestra.run.smithi189.stdout:Architecture: amd64 2024-03-07T02:37:27.804 INFO:teuthology.orchestra.run.smithi189.stdout:Source: linux-meta 2024-03-07T02:37:27.804 INFO:teuthology.orchestra.run.smithi189.stdout:Version: 5.15.0.97.92 2024-03-07T02:37:27.804 INFO:teuthology.orchestra.run.smithi189.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.2), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-97), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.2) 2024-03-07T02:37:27.804 INFO:teuthology.orchestra.run.smithi189.stdout:Depends: linux-image-5.15.0-97-generic, linux-modules-extra-5.15.0-97-generic, linux-firmware, intel-microcode, amd64-microcode 2024-03-07T02:37:27.804 INFO:teuthology.orchestra.run.smithi189.stdout:Recommends: thermald 2024-03-07T02:37:27.804 INFO:teuthology.orchestra.run.smithi189.stdout:Description: Generic Linux kernel image 2024-03-07T02:37:27.804 INFO:teuthology.orchestra.run.smithi189.stdout: This package will always depend on the latest generic kernel image 2024-03-07T02:37:27.804 INFO:teuthology.orchestra.run.smithi189.stdout: available. 2024-03-07T02:37:27.805 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-97-generic 2024-03-07T02:37:27.805 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-07T02:37:27.805 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-07T02:37:27.805 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-03-07T02:37:27.805 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi189.front.sepia.ceph.com, path=None, version=distro) 2024-03-07T02:37:27.805 DEBUG:teuthology.orchestra.run.smithi189:> sudo apt-get clean 2024-03-07T02:37:27.987 DEBUG:teuthology.orchestra.run.smithi189:> sudo apt-get update 2024-03-07T02:37:28.137 INFO:teuthology.orchestra.run.smithi189.stdout:Hit:1 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-03-07T02:37:28.228 INFO:teuthology.orchestra.run.smithi189.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-03-07T02:37:28.310 INFO:teuthology.orchestra.run.smithi189.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-03-07T02:37:28.391 INFO:teuthology.orchestra.run.smithi189.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-03-07T02:37:29.287 INFO:teuthology.orchestra.run.smithi145.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-03-07T02:37:29.287 INFO:teuthology.orchestra.run.smithi145.stdout:Sourcing file `/etc/default/grub' 2024-03-07T02:37:29.306 INFO:teuthology.orchestra.run.smithi145.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-03-07T02:37:29.328 INFO:teuthology.orchestra.run.smithi145.stdout:Generating grub configuration file ... 2024-03-07T02:37:29.504 INFO:teuthology.orchestra.run.smithi189.stdout:Reading package lists... 2024-03-07T02:37:29.522 DEBUG:teuthology.orchestra.run.smithi189:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-03-07T02:37:29.581 INFO:teuthology.orchestra.run.smithi189.stdout:Reading package lists... 2024-03-07T02:37:29.599 INFO:teuthology.orchestra.run.smithi145.stdout:Found linux image: /boot/vmlinuz-5.15.0-97-generic 2024-03-07T02:37:29.621 INFO:teuthology.orchestra.run.smithi145.stdout:Found initrd image: /boot/initrd.img-5.15.0-97-generic 2024-03-07T02:37:29.776 INFO:teuthology.orchestra.run.smithi189.stdout:Building dependency tree... 2024-03-07T02:37:29.777 INFO:teuthology.orchestra.run.smithi189.stdout:Reading state information... 2024-03-07T02:37:29.926 INFO:teuthology.orchestra.run.smithi145.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-03-07T02:37:29.932 INFO:teuthology.orchestra.run.smithi145.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-03-07T02:37:29.947 INFO:teuthology.orchestra.run.smithi189.stdout:linux-image-generic is already the newest version (5.15.0.97.92). 2024-03-07T02:37:29.947 INFO:teuthology.orchestra.run.smithi189.stdout:The following packages were automatically installed and are no longer required: 2024-03-07T02:37:29.948 INFO:teuthology.orchestra.run.smithi189.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-03-07T02:37:29.948 INFO:teuthology.orchestra.run.smithi189.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-03-07T02:37:29.948 INFO:teuthology.orchestra.run.smithi189.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-03-07T02:37:29.948 INFO:teuthology.orchestra.run.smithi189.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-03-07T02:37:29.948 INFO:teuthology.orchestra.run.smithi189.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-03-07T02:37:29.949 INFO:teuthology.orchestra.run.smithi189.stdout:Use 'sudo apt autoremove' to remove them. 2024-03-07T02:37:29.990 INFO:teuthology.orchestra.run.smithi189.stdout:0 upgraded, 0 newly installed, 0 to remove and 288 not upgraded. 2024-03-07T02:37:29.992 DEBUG:teuthology.orchestra.run.smithi189:> dpkg -s linux-image-generic 2024-03-07T02:37:30.006 INFO:teuthology.orchestra.run.smithi145.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-03-07T02:37:30.007 INFO:teuthology.orchestra.run.smithi189.stdout:Package: linux-image-generic 2024-03-07T02:37:30.008 INFO:teuthology.orchestra.run.smithi189.stdout:Status: install ok installed 2024-03-07T02:37:30.008 INFO:teuthology.orchestra.run.smithi189.stdout:Priority: optional 2024-03-07T02:37:30.008 INFO:teuthology.orchestra.run.smithi189.stdout:Section: kernel 2024-03-07T02:37:30.008 INFO:teuthology.orchestra.run.smithi189.stdout:Installed-Size: 21 2024-03-07T02:37:30.008 INFO:teuthology.orchestra.run.smithi189.stdout:Maintainer: Ubuntu Kernel Team 2024-03-07T02:37:30.008 INFO:teuthology.orchestra.run.smithi189.stdout:Architecture: amd64 2024-03-07T02:37:30.008 INFO:teuthology.orchestra.run.smithi189.stdout:Source: linux-meta 2024-03-07T02:37:30.008 INFO:teuthology.orchestra.run.smithi189.stdout:Version: 5.15.0.97.92 2024-03-07T02:37:30.008 INFO:teuthology.orchestra.run.smithi189.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.2), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-97), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.2) 2024-03-07T02:37:30.008 INFO:teuthology.orchestra.run.smithi189.stdout:Depends: linux-image-5.15.0-97-generic, linux-modules-extra-5.15.0-97-generic, linux-firmware, intel-microcode, amd64-microcode 2024-03-07T02:37:30.008 INFO:teuthology.orchestra.run.smithi189.stdout:Recommends: thermald 2024-03-07T02:37:30.009 INFO:teuthology.orchestra.run.smithi189.stdout:Description: Generic Linux kernel image 2024-03-07T02:37:30.009 INFO:teuthology.orchestra.run.smithi189.stdout: This package will always depend on the latest generic kernel image 2024-03-07T02:37:30.009 INFO:teuthology.orchestra.run.smithi189.stdout: available. 2024-03-07T02:37:30.009 DEBUG:teuthology.orchestra.run.smithi189:> mktemp 2024-03-07T02:37:30.012 INFO:teuthology.orchestra.run.smithi145.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-03-07T02:37:30.056 INFO:teuthology.orchestra.run.smithi189.stdout:/tmp/tmp.1ztMORhbr8 2024-03-07T02:37:30.057 DEBUG:teuthology.orchestra.run.smithi189:> sudo cp /boot/grub/grub.cfg /tmp/tmp.1ztMORhbr8 2024-03-07T02:37:30.112 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod 0666 /tmp/tmp.1ztMORhbr8 2024-03-07T02:37:30.183 INFO:teuthology.orchestra.run.smithi145.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-03-07T02:37:30.183 INFO:teuthology.orchestra.run.smithi145.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-03-07T02:37:30.183 INFO:teuthology.orchestra.run.smithi145.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-03-07T02:37:30.221 INFO:teuthology.orchestra.run.smithi145.stdout:done 2024-03-07T02:37:30.231 DEBUG:teuthology.orchestra.remote:smithi189:/tmp/tmp.1ztMORhbr8 is 10KB 2024-03-07T02:37:30.243 DEBUG:teuthology.orchestra.run.smithi189:> rm -fr /tmp/tmp.1ztMORhbr8 2024-03-07T02:37:30.251 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-03-07T02:37:30.305 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-07T02:37:30.305 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-03-07T02:37:30.305 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-03-07T02:37:30.371 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-97-generic 2024-03-07T02:37:30.371 DEBUG:teuthology.orchestra.run.smithi189:> sudo update-grub 2024-03-07T02:37:30.665 DEBUG:teuthology.orchestra.run.smithi145:> dpkg -s linux-image-generic 2024-03-07T02:37:30.689 INFO:teuthology.orchestra.run.smithi145.stdout:Package: linux-image-generic 2024-03-07T02:37:30.689 INFO:teuthology.orchestra.run.smithi145.stdout:Status: install ok installed 2024-03-07T02:37:30.689 INFO:teuthology.orchestra.run.smithi145.stdout:Priority: optional 2024-03-07T02:37:30.689 INFO:teuthology.orchestra.run.smithi145.stdout:Section: kernel 2024-03-07T02:37:30.689 INFO:teuthology.orchestra.run.smithi145.stdout:Installed-Size: 21 2024-03-07T02:37:30.689 INFO:teuthology.orchestra.run.smithi145.stdout:Maintainer: Ubuntu Kernel Team 2024-03-07T02:37:30.689 INFO:teuthology.orchestra.run.smithi145.stdout:Architecture: amd64 2024-03-07T02:37:30.689 INFO:teuthology.orchestra.run.smithi145.stdout:Source: linux-meta 2024-03-07T02:37:30.689 INFO:teuthology.orchestra.run.smithi145.stdout:Version: 5.15.0.97.92 2024-03-07T02:37:30.690 INFO:teuthology.orchestra.run.smithi145.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.2), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-97), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.2) 2024-03-07T02:37:30.690 INFO:teuthology.orchestra.run.smithi145.stdout:Depends: linux-image-5.15.0-97-generic, linux-modules-extra-5.15.0-97-generic, linux-firmware, intel-microcode, amd64-microcode 2024-03-07T02:37:30.690 INFO:teuthology.orchestra.run.smithi145.stdout:Recommends: thermald 2024-03-07T02:37:30.690 INFO:teuthology.orchestra.run.smithi145.stdout:Description: Generic Linux kernel image 2024-03-07T02:37:30.690 INFO:teuthology.orchestra.run.smithi145.stdout: This package will always depend on the latest generic kernel image 2024-03-07T02:37:30.690 INFO:teuthology.orchestra.run.smithi145.stdout: available. 2024-03-07T02:37:30.690 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-97-generic 2024-03-07T02:37:30.690 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-07T02:37:30.690 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-07T02:37:30.690 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-03-07T02:37:30.690 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi145.front.sepia.ceph.com, path=None, version=distro) 2024-03-07T02:37:30.691 DEBUG:teuthology.orchestra.run.smithi145:> sudo apt-get clean 2024-03-07T02:37:30.840 DEBUG:teuthology.orchestra.run.smithi145:> sudo apt-get update 2024-03-07T02:37:30.964 INFO:teuthology.orchestra.run.smithi145.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-03-07T02:37:30.977 INFO:teuthology.orchestra.run.smithi145.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-03-07T02:37:30.998 INFO:teuthology.orchestra.run.smithi145.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-03-07T02:37:31.079 INFO:teuthology.orchestra.run.smithi145.stdout:Hit:4 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-03-07T02:37:31.565 INFO:teuthology.orchestra.run.smithi189.stderr:Sourcing file `/etc/default/grub' 2024-03-07T02:37:31.566 INFO:teuthology.orchestra.run.smithi189.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-03-07T02:37:31.569 INFO:teuthology.orchestra.run.smithi189.stderr:Generating grub configuration file ... 2024-03-07T02:37:31.732 INFO:teuthology.orchestra.run.smithi189.stderr:Found linux image: /boot/vmlinuz-5.15.0-97-generic 2024-03-07T02:37:31.739 INFO:teuthology.orchestra.run.smithi189.stderr:Found initrd image: /boot/initrd.img-5.15.0-97-generic 2024-03-07T02:37:31.950 INFO:teuthology.orchestra.run.smithi189.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-03-07T02:37:31.955 INFO:teuthology.orchestra.run.smithi189.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-03-07T02:37:31.998 INFO:teuthology.orchestra.run.smithi189.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-03-07T02:37:32.003 INFO:teuthology.orchestra.run.smithi189.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-03-07T02:37:32.125 INFO:teuthology.orchestra.run.smithi189.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-03-07T02:37:32.125 INFO:teuthology.orchestra.run.smithi189.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-03-07T02:37:32.125 INFO:teuthology.orchestra.run.smithi189.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-03-07T02:37:32.140 INFO:teuthology.orchestra.run.smithi189.stderr:done 2024-03-07T02:37:32.149 DEBUG:teuthology.orchestra.run.smithi189:> sudo shutdown -r now 2024-03-07T02:37:32.212 INFO:teuthology.orchestra.run.smithi145.stdout:Reading package lists... 2024-03-07T02:37:32.230 DEBUG:teuthology.orchestra.run.smithi145:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-03-07T02:37:32.292 INFO:teuthology.orchestra.run.smithi145.stdout:Reading package lists... 2024-03-07T02:37:32.487 INFO:teuthology.orchestra.run.smithi145.stdout:Building dependency tree... 2024-03-07T02:37:32.487 INFO:teuthology.orchestra.run.smithi145.stdout:Reading state information... 2024-03-07T02:37:32.655 INFO:teuthology.orchestra.run.smithi145.stdout:linux-image-generic is already the newest version (5.15.0.97.92). 2024-03-07T02:37:32.656 INFO:teuthology.orchestra.run.smithi145.stdout:The following packages were automatically installed and are no longer required: 2024-03-07T02:37:32.656 INFO:teuthology.orchestra.run.smithi145.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-03-07T02:37:32.656 INFO:teuthology.orchestra.run.smithi145.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-03-07T02:37:32.656 INFO:teuthology.orchestra.run.smithi145.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-03-07T02:37:32.656 INFO:teuthology.orchestra.run.smithi145.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-03-07T02:37:32.657 INFO:teuthology.orchestra.run.smithi145.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-03-07T02:37:32.657 INFO:teuthology.orchestra.run.smithi145.stdout:Use 'sudo apt autoremove' to remove them. 2024-03-07T02:37:32.696 INFO:teuthology.orchestra.run.smithi145.stdout:0 upgraded, 0 newly installed, 0 to remove and 288 not upgraded. 2024-03-07T02:37:32.698 DEBUG:teuthology.orchestra.run.smithi145:> dpkg -s linux-image-generic 2024-03-07T02:37:32.714 INFO:teuthology.orchestra.run.smithi145.stdout:Package: linux-image-generic 2024-03-07T02:37:32.714 INFO:teuthology.orchestra.run.smithi145.stdout:Status: install ok installed 2024-03-07T02:37:32.714 INFO:teuthology.orchestra.run.smithi145.stdout:Priority: optional 2024-03-07T02:37:32.714 INFO:teuthology.orchestra.run.smithi145.stdout:Section: kernel 2024-03-07T02:37:32.715 INFO:teuthology.orchestra.run.smithi145.stdout:Installed-Size: 21 2024-03-07T02:37:32.715 INFO:teuthology.orchestra.run.smithi145.stdout:Maintainer: Ubuntu Kernel Team 2024-03-07T02:37:32.715 INFO:teuthology.orchestra.run.smithi145.stdout:Architecture: amd64 2024-03-07T02:37:32.715 INFO:teuthology.orchestra.run.smithi145.stdout:Source: linux-meta 2024-03-07T02:37:32.715 INFO:teuthology.orchestra.run.smithi145.stdout:Version: 5.15.0.97.92 2024-03-07T02:37:32.715 INFO:teuthology.orchestra.run.smithi145.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.2), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-97), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.2) 2024-03-07T02:37:32.715 INFO:teuthology.orchestra.run.smithi145.stdout:Depends: linux-image-5.15.0-97-generic, linux-modules-extra-5.15.0-97-generic, linux-firmware, intel-microcode, amd64-microcode 2024-03-07T02:37:32.715 INFO:teuthology.orchestra.run.smithi145.stdout:Recommends: thermald 2024-03-07T02:37:32.715 INFO:teuthology.orchestra.run.smithi145.stdout:Description: Generic Linux kernel image 2024-03-07T02:37:32.715 INFO:teuthology.orchestra.run.smithi145.stdout: This package will always depend on the latest generic kernel image 2024-03-07T02:37:32.715 INFO:teuthology.orchestra.run.smithi145.stdout: available. 2024-03-07T02:37:32.716 DEBUG:teuthology.orchestra.run.smithi145:> mktemp 2024-03-07T02:37:32.763 INFO:teuthology.orchestra.run.smithi145.stdout:/tmp/tmp.hIzns3KvF9 2024-03-07T02:37:32.764 DEBUG:teuthology.orchestra.run.smithi145:> sudo cp /boot/grub/grub.cfg /tmp/tmp.hIzns3KvF9 2024-03-07T02:37:32.819 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod 0666 /tmp/tmp.hIzns3KvF9 2024-03-07T02:37:32.939 DEBUG:teuthology.orchestra.remote:smithi145:/tmp/tmp.hIzns3KvF9 is 10KB 2024-03-07T02:37:32.953 DEBUG:teuthology.orchestra.run.smithi145:> rm -fr /tmp/tmp.hIzns3KvF9 2024-03-07T02:37:32.958 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-03-07T02:37:33.010 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:37:33.010 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-03-07T02:37:33.010 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-03-07T02:37:33.088 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-97-generic 2024-03-07T02:37:33.088 DEBUG:teuthology.orchestra.run.smithi145:> sudo update-grub 2024-03-07T02:37:34.221 INFO:teuthology.orchestra.run.smithi145.stderr:Sourcing file `/etc/default/grub' 2024-03-07T02:37:34.221 INFO:teuthology.orchestra.run.smithi145.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-03-07T02:37:34.224 INFO:teuthology.orchestra.run.smithi145.stderr:Generating grub configuration file ... 2024-03-07T02:37:34.422 INFO:teuthology.orchestra.run.smithi145.stderr:Found linux image: /boot/vmlinuz-5.15.0-97-generic 2024-03-07T02:37:34.429 INFO:teuthology.orchestra.run.smithi145.stderr:Found initrd image: /boot/initrd.img-5.15.0-97-generic 2024-03-07T02:37:34.673 INFO:teuthology.orchestra.run.smithi145.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-03-07T02:37:34.678 INFO:teuthology.orchestra.run.smithi145.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-03-07T02:37:34.715 INFO:teuthology.orchestra.run.smithi145.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-03-07T02:37:34.719 INFO:teuthology.orchestra.run.smithi145.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-03-07T02:37:34.839 INFO:teuthology.orchestra.run.smithi145.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-03-07T02:37:34.840 INFO:teuthology.orchestra.run.smithi145.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-03-07T02:37:34.840 INFO:teuthology.orchestra.run.smithi145.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-03-07T02:37:34.857 INFO:teuthology.orchestra.run.smithi145.stderr:done 2024-03-07T02:37:34.869 DEBUG:teuthology.orchestra.run.smithi145:> sudo shutdown -r now 2024-03-07T02:38:02.176 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-03-07T02:38:02.177 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-03-07T02:38:02.178 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:38:04.874 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-03-07T02:38:04.874 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-03-07T02:38:04.875 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:38:36.860 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.189 2024-03-07T02:38:45.868 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-03-07T02:38:45.869 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:39:04.896 DEBUG:teuthology.orchestra.remote:timed out 2024-03-07T02:39:13.905 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-03-07T02:39:13.906 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:39:13.908 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.145 2024-03-07T02:39:25.920 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-03-07T02:39:25.921 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:39:26.212 DEBUG:teuthology.orchestra.run.smithi145:> true 2024-03-07T02:39:26.740 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-03-07T02:39:26.740 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-97-generic"... 2024-03-07T02:39:26.741 DEBUG:teuthology.orchestra.run.smithi145:> uname -r 2024-03-07T02:39:26.788 INFO:teuthology.orchestra.run.smithi145.stdout:5.15.0-97-generic 2024-03-07T02:39:26.788 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-97-generic vs 5.15.0-97-generic 2024-03-07T02:39:26.788 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-07T02:39:26.789 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-03-07T02:39:27.789 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-03-07T02:39:27.789 DEBUG:teuthology.orchestra.run.smithi145:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-07T02:39:28.116 INFO:teuthology.orchestra.run.smithi145.stdout:ttyS1 2024-03-07T02:39:28.150 DEBUG:teuthology.parallel:result is None 2024-03-07T02:39:45.872 DEBUG:teuthology.orchestra.remote:timed out 2024-03-07T02:39:57.873 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-03-07T02:39:57.874 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:39:58.348 DEBUG:teuthology.orchestra.run.smithi189:> true 2024-03-07T02:39:58.884 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-03-07T02:39:58.884 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-97-generic"... 2024-03-07T02:39:58.884 DEBUG:teuthology.orchestra.run.smithi189:> uname -r 2024-03-07T02:39:58.929 INFO:teuthology.orchestra.run.smithi189.stdout:5.15.0-97-generic 2024-03-07T02:39:58.929 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-97-generic vs 5.15.0-97-generic 2024-03-07T02:39:58.930 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-07T02:39:58.930 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-03-07T02:39:59.930 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-03-07T02:39:59.931 DEBUG:teuthology.orchestra.run.smithi189:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-07T02:40:00.334 INFO:teuthology.orchestra.run.smithi189.stdout:ttyS1 2024-03-07T02:40:00.360 DEBUG:teuthology.parallel:result is None 2024-03-07T02:40:00.360 INFO:teuthology.run_tasks:Running task internal.base... 2024-03-07T02:40:00.369 INFO:teuthology.task.internal:Creating test directory... 2024-03-07T02:40:00.369 DEBUG:teuthology.orchestra.run.smithi145:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-07T02:40:00.372 DEBUG:teuthology.orchestra.run.smithi189:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-07T02:40:00.378 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-03-07T02:40:00.425 INFO:teuthology.run_tasks:Running task internal.archive... 2024-03-07T02:40:00.457 INFO:teuthology.task.internal:Creating archive directory... 2024-03-07T02:40:00.458 DEBUG:teuthology.orchestra.run.smithi145:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-07T02:40:00.460 DEBUG:teuthology.orchestra.run.smithi189:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-07T02:40:00.564 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-03-07T02:40:00.577 INFO:teuthology.task.internal:Enabling coredump saving... 2024-03-07T02:40:00.577 DEBUG:teuthology.orchestra.run.smithi145:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-07T02:40:00.581 DEBUG:teuthology.orchestra.run.smithi189:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-07T02:40:00.596 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-07T02:40:00.609 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-07T02:40:00.618 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-07T02:40:00.628 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-07T02:40:00.629 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-03-07T02:40:00.637 INFO:teuthology.task.internal:Configuring sudo... 2024-03-07T02:40:00.637 DEBUG:teuthology.orchestra.run.smithi145:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-07T02:40:00.653 DEBUG:teuthology.orchestra.run.smithi189:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-07T02:40:00.688 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-03-07T02:40:00.703 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-03-07T02:40:00.703 DEBUG:teuthology.orchestra.run.smithi145:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-07T02:40:00.709 DEBUG:teuthology.orchestra.run.smithi189:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-07T02:40:00.747 DEBUG:teuthology.orchestra.run.smithi145:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-07T02:40:00.760 DEBUG:teuthology.orchestra.run.smithi145:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-07T02:40:00.808 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:40:00.808 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-07T02:40:00.867 DEBUG:teuthology.orchestra.run.smithi189:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-07T02:40:00.880 DEBUG:teuthology.orchestra.run.smithi189:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-07T02:40:00.930 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-07T02:40:00.930 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-07T02:40:01.012 DEBUG:teuthology.orchestra.run.smithi145:> sudo service rsyslog restart 2024-03-07T02:40:01.016 DEBUG:teuthology.orchestra.run.smithi189:> sudo service rsyslog restart 2024-03-07T02:40:01.084 INFO:teuthology.run_tasks:Running task internal.timer... 2024-03-07T02:40:01.091 INFO:teuthology.task.internal:Starting timer... 2024-03-07T02:40:01.091 INFO:teuthology.run_tasks:Running task pcp... 2024-03-07T02:40:01.167 INFO:teuthology.run_tasks:Running task selinux... 2024-03-07T02:40:01.233 DEBUG:teuthology.task.selinux:Excluding smithi145: OS 'ubuntu' does not support SELinux 2024-03-07T02:40:01.233 DEBUG:teuthology.task.selinux:Excluding smithi189: OS 'ubuntu' does not support SELinux 2024-03-07T02:40:01.233 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-03-07T02:40:01.233 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2024-03-07T02:40:01.233 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-03-07T02:40:01.233 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-03-07T02:40:01.266 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-03-07T02:40:01.302 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-03-07T02:40:01.705 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-03-07T02:40:01.813 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-03-07T02:40:01.814 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi145.front.sepia.ceph.com,smithi189.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-03-07T02:49:07.080 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi145.front.sepia.ceph.com'), Remote(name='ubuntu@smithi189.front.sepia.ceph.com')] 2024-03-07T02:49:07.081 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-03-07T02:49:07.082 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi145.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:49:07.161 DEBUG:teuthology.orchestra.run.smithi145:> true 2024-03-07T02:49:07.236 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi145.front.sepia.ceph.com' 2024-03-07T02:49:07.236 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-03-07T02:49:07.237 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:49:07.313 DEBUG:teuthology.orchestra.run.smithi189:> true 2024-03-07T02:49:07.387 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-03-07T02:49:07.388 INFO:teuthology.run_tasks:Running task clock... 2024-03-07T02:49:07.404 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-03-07T02:49:07.404 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-07T02:49:07.405 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-07T02:49:07.407 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-07T02:49:07.407 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-07T02:49:07.431 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-03-07T02:49:07.432 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: Command line: ntpd -gq 2024-03-07T02:49:07.432 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: ---------------------------------------------------- 2024-03-07T02:49:07.432 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: ntp-4 is maintained by Network Time Foundation, 2024-03-07T02:49:07.432 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-03-07T02:49:07.432 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: corporation. Support and training for ntp-4 are 2024-03-07T02:49:07.432 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: available at https://www.nwtime.org/support 2024-03-07T02:49:07.432 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: ---------------------------------------------------- 2024-03-07T02:49:07.433 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: proto: precision = 0.060 usec (-24) 2024-03-07T02:49:07.433 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: basedate set to 2022-02-04 2024-03-07T02:49:07.433 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: gps base set to 2022-02-06 (week 2196) 2024-03-07T02:49:07.433 INFO:teuthology.orchestra.run.smithi145.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-03-07T02:49:07.434 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-03-07T02:49:07.434 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: restrict ::: KOD does nothing without LIMITED. 2024-03-07T02:49:07.434 INFO:teuthology.orchestra.run.smithi145.stderr:restrict ::: KOD does nothing without LIMITED. 2024-03-07T02:49:07.435 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: Listen and drop on 0 v6wildcard [::]:123 2024-03-07T02:49:07.435 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-03-07T02:49:07.435 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: Listen normally on 2 lo 127.0.0.1:123 2024-03-07T02:49:07.435 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: Listen normally on 3 enp3s0f1 172.21.15.145:123 2024-03-07T02:49:07.435 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: Listen normally on 4 lo [::1]:123 2024-03-07T02:49:07.435 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:7791%5]:123 2024-03-07T02:49:07.436 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:07 ntpd[17174]: Listening on routing socket on fd #22 for interface updates 2024-03-07T02:49:07.461 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-03-07T02:49:07.462 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: Command line: ntpd -gq 2024-03-07T02:49:07.462 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: ---------------------------------------------------- 2024-03-07T02:49:07.462 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: ntp-4 is maintained by Network Time Foundation, 2024-03-07T02:49:07.462 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-03-07T02:49:07.462 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: corporation. Support and training for ntp-4 are 2024-03-07T02:49:07.462 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: available at https://www.nwtime.org/support 2024-03-07T02:49:07.462 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: ---------------------------------------------------- 2024-03-07T02:49:07.462 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: proto: precision = 0.064 usec (-24) 2024-03-07T02:49:07.462 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: basedate set to 2022-02-04 2024-03-07T02:49:07.462 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: gps base set to 2022-02-06 (week 2196) 2024-03-07T02:49:07.462 INFO:teuthology.orchestra.run.smithi189.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-03-07T02:49:07.463 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-03-07T02:49:07.463 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: restrict ::: KOD does nothing without LIMITED. 2024-03-07T02:49:07.463 INFO:teuthology.orchestra.run.smithi189.stderr:restrict ::: KOD does nothing without LIMITED. 2024-03-07T02:49:07.464 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: Listen and drop on 0 v6wildcard [::]:123 2024-03-07T02:49:07.464 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-03-07T02:49:07.464 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: Listen normally on 2 lo 127.0.0.1:123 2024-03-07T02:49:07.464 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: Listen normally on 3 enp3s0f1 172.21.15.189:123 2024-03-07T02:49:07.464 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: Listen normally on 4 lo [::1]:123 2024-03-07T02:49:07.464 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:6df1%5]:123 2024-03-07T02:49:07.464 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:07 ntpd[17143]: Listening on routing socket on fd #22 for interface updates 2024-03-07T02:49:08.432 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:08 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:08.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:08 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:08.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:08 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:08.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:08 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:08.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:08 ntpd[17174]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:08.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:08 ntpd[17174]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:08.461 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:08 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:08.461 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:08 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:09.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:09 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:09.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:09 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:09.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:09 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:09.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:09 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:09.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:09 ntpd[17143]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:09.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:09 ntpd[17143]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:10.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:10 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:10.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:10 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:10.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:10 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:10.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:10 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:10.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:10 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:10.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:10 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:10.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:10 ntpd[17174]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:10.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:10 ntpd[17174]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:10.434 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:10 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:10.434 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:10 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:10.434 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:10 ntpd[17174]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:10.434 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:10 ntpd[17174]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:11.432 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:11 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:11.432 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:11 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:11.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:11 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:11.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:11 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:11.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:11 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:11.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:11 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:11.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:11 ntpd[17174]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:11.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:11 ntpd[17174]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:11.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:11 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:11.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:11 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:11.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:11 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:11.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:11 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:11.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:11 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:11.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:11 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:11.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:11 ntpd[17143]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:11.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:11 ntpd[17143]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:11.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:11 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:11.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:11 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:11.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:11 ntpd[17143]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:11.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:11 ntpd[17143]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:12.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:12 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:12.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:12 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:12.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:12 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:12.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:12 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:12.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:12 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:12.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:12 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:12.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:12 ntpd[17174]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:12.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:12 ntpd[17174]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:12.434 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:12 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:12.434 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:12 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:12.434 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:12 ntpd[17174]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:12.434 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:12 ntpd[17174]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:12.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:12 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:12.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:12 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:12.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:12 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:12.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:12 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:12.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:12 ntpd[17143]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:12.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:12 ntpd[17143]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:13.432 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:13 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:13.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:13 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:13.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:13 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:13.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:13 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:13.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:13 ntpd[17174]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:13.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:13 ntpd[17174]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:13.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:13 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:13.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:13 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:13.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:13 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:13.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:13 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:13.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:13 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:13.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:13 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:13.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:13 ntpd[17143]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:13.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:13 ntpd[17143]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:13.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:13 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:13.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:13 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:13.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:13 ntpd[17143]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:13.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:13 ntpd[17143]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:14.432 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:14 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:14.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:14 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:14.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:14 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:14.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:14 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:14.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:14 ntpd[17174]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:14.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:14 ntpd[17174]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:14.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:14 ntpd[17174]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:14.433 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:14 ntpd[17174]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:14.433 INFO:teuthology.orchestra.run.smithi145.stdout: 7 Mar 02:49:14 ntpd[17174]: ntpd: time slew +0.000612 s 2024-03-07T02:49:14.434 INFO:teuthology.orchestra.run.smithi145.stdout:ntpd: time slew +0.000612s 2024-03-07T02:49:14.434 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:14 ntpd[17174]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-03-07T02:49:14.434 INFO:teuthology.orchestra.run.smithi145.stderr: 7 Mar 02:49:14 ntpd[17174]: can't open /var/log/ntpstats/loopstats.20240307: Permission denied 2024-03-07T02:49:14.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:14 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:14.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:14 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:14.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:14 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:14.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:14 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:14.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:14 ntpd[17143]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:14.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:14 ntpd[17143]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:14.482 INFO:teuthology.orchestra.run.smithi145.stdout: remote refid st t when poll reach delay offset jitter 2024-03-07T02:49:14.483 INFO:teuthology.orchestra.run.smithi145.stdout:============================================================================== 2024-03-07T02:49:14.483 INFO:teuthology.orchestra.run.smithi145.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:49:14.483 INFO:teuthology.orchestra.run.smithi145.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:49:14.483 INFO:teuthology.orchestra.run.smithi145.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:49:14.483 INFO:teuthology.orchestra.run.smithi145.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:49:15.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:15.462 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:15.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:15.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:15.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:15.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:15.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:15.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:15.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:15.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:15.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:15.463 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:15.463 INFO:teuthology.orchestra.run.smithi189.stdout: 7 Mar 02:49:15 ntpd[17143]: ntpd: time slew -0.004818 s 2024-03-07T02:49:15.464 INFO:teuthology.orchestra.run.smithi189.stdout:ntpd: time slew -0.004818s 2024-03-07T02:49:15.464 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-03-07T02:49:15.464 INFO:teuthology.orchestra.run.smithi189.stderr: 7 Mar 02:49:15 ntpd[17143]: can't open /var/log/ntpstats/loopstats.20240307: Permission denied 2024-03-07T02:49:15.508 INFO:teuthology.orchestra.run.smithi189.stdout: remote refid st t when poll reach delay offset jitter 2024-03-07T02:49:15.508 INFO:teuthology.orchestra.run.smithi189.stdout:============================================================================== 2024-03-07T02:49:15.508 INFO:teuthology.orchestra.run.smithi189.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:49:15.508 INFO:teuthology.orchestra.run.smithi189.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:49:15.508 INFO:teuthology.orchestra.run.smithi189.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:49:15.508 INFO:teuthology.orchestra.run.smithi189.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:49:15.509 INFO:teuthology.run_tasks:Running task install... 2024-03-07T02:49:15.517 DEBUG:teuthology.task.install:project ceph 2024-03-07T02:49:15.517 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'ea9a5688d9980e4ddedb59679d2960bcbac6fc71'}} 2024-03-07T02:49:15.518 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'ea9a5688d9980e4ddedb59679d2960bcbac6fc71'} 2024-03-07T02:49:15.518 INFO:teuthology.task.install:Using flavor: default 2024-03-07T02:49:15.523 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-03-07T02:49:15.523 INFO:teuthology.task.install:extra packages: [] 2024-03-07T02:49:15.524 DEBUG:teuthology.orchestra.run.smithi145:> sudo apt-key list | grep Ceph 2024-03-07T02:49:15.524 DEBUG:teuthology.orchestra.run.smithi189:> sudo apt-key list | grep Ceph 2024-03-07T02:49:15.589 INFO:teuthology.orchestra.run.smithi145.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-03-07T02:49:15.617 INFO:teuthology.orchestra.run.smithi145.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-03-07T02:49:15.618 INFO:teuthology.orchestra.run.smithi145.stdout:uid [ unknown] Ceph.com (release key) 2024-03-07T02:49:15.619 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-03-07T02:49:15.619 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-07T02:49:15.619 INFO:teuthology.packaging:ref: None 2024-03-07T02:49:15.619 INFO:teuthology.packaging:tag: None 2024-03-07T02:49:15.619 INFO:teuthology.packaging:branch: reef 2024-03-07T02:49:15.619 INFO:teuthology.packaging:sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T02:49:15.619 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-03-07T02:49:15.636 INFO:teuthology.orchestra.run.smithi189.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-03-07T02:49:15.772 INFO:teuthology.orchestra.run.smithi189.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-03-07T02:49:15.773 INFO:teuthology.orchestra.run.smithi189.stdout:uid [ unknown] Ceph.com (release key) 2024-03-07T02:49:15.773 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-03-07T02:49:15.773 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-07T02:49:15.773 INFO:teuthology.packaging:ref: None 2024-03-07T02:49:15.773 INFO:teuthology.packaging:tag: None 2024-03-07T02:49:15.773 INFO:teuthology.packaging:branch: reef 2024-03-07T02:49:15.773 INFO:teuthology.packaging:sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T02:49:15.773 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-03-07T02:49:15.955 INFO:teuthology.task.install.deb:Pulling from https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default/ 2024-03-07T02:49:15.956 INFO:teuthology.task.install.deb:Package version is 18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:49:15.990 INFO:teuthology.task.install.deb:Pulling from https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default/ 2024-03-07T02:49:15.991 INFO:teuthology.task.install.deb:Package version is 18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:49:16.126 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:49:16.126 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-03-07T02:49:16.139 DEBUG:teuthology.orchestra.run.smithi145:> sudo apt-get update 2024-03-07T02:49:16.161 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-07T02:49:16.161 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-03-07T02:49:16.175 DEBUG:teuthology.orchestra.run.smithi189:> sudo apt-get update 2024-03-07T02:49:16.312 INFO:teuthology.orchestra.run.smithi145.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-03-07T02:49:16.325 INFO:teuthology.orchestra.run.smithi145.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-03-07T02:49:16.346 INFO:teuthology.orchestra.run.smithi145.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-03-07T02:49:16.361 INFO:teuthology.orchestra.run.smithi145.stdout:Ign:4 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy InRelease 2024-03-07T02:49:16.384 INFO:teuthology.orchestra.run.smithi145.stdout:Get:5 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy Release [7,662 B] 2024-03-07T02:49:16.388 INFO:teuthology.orchestra.run.smithi145.stdout:Hit:6 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-03-07T02:49:16.405 INFO:teuthology.orchestra.run.smithi145.stdout:Ign:7 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy Release.gpg 2024-03-07T02:49:16.426 INFO:teuthology.orchestra.run.smithi145.stdout:Get:8 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.5 kB] 2024-03-07T02:49:16.453 INFO:teuthology.orchestra.run.smithi145.stdout:Get:9 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,409 B] 2024-03-07T02:49:16.457 INFO:teuthology.orchestra.run.smithi189.stdout:Ign:1 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy InRelease 2024-03-07T02:49:16.470 INFO:teuthology.orchestra.run.smithi189.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-03-07T02:49:16.494 INFO:teuthology.orchestra.run.smithi189.stdout:Get:3 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy Release [7,662 B] 2024-03-07T02:49:16.510 INFO:teuthology.orchestra.run.smithi189.stdout:Hit:4 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-03-07T02:49:16.532 INFO:teuthology.orchestra.run.smithi189.stdout:Ign:5 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy Release.gpg 2024-03-07T02:49:16.543 INFO:teuthology.orchestra.run.smithi189.stdout:Hit:6 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-03-07T02:49:16.568 INFO:teuthology.orchestra.run.smithi189.stdout:Get:7 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,409 B] 2024-03-07T02:49:16.611 INFO:teuthology.orchestra.run.smithi189.stdout:Get:8 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.5 kB] 2024-03-07T02:49:16.628 INFO:teuthology.orchestra.run.smithi189.stdout:Hit:9 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-03-07T02:49:16.641 INFO:teuthology.orchestra.run.smithi145.stdout:Fetched 28.5 kB in 0s (87.7 kB/s) 2024-03-07T02:49:16.846 INFO:teuthology.orchestra.run.smithi189.stdout:Fetched 28.5 kB in 0s (59.2 kB/s) 2024-03-07T02:49:17.598 INFO:teuthology.orchestra.run.smithi145.stdout:Reading package lists... 2024-03-07T02:49:17.613 DEBUG:teuthology.orchestra.run.smithi145:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=18.2.1-707-gffcfe7db-1jammy cephadm=18.2.1-707-gffcfe7db-1jammy ceph-mds=18.2.1-707-gffcfe7db-1jammy ceph-mgr=18.2.1-707-gffcfe7db-1jammy ceph-common=18.2.1-707-gffcfe7db-1jammy ceph-fuse=18.2.1-707-gffcfe7db-1jammy ceph-test=18.2.1-707-gffcfe7db-1jammy radosgw=18.2.1-707-gffcfe7db-1jammy python3-rados=18.2.1-707-gffcfe7db-1jammy python3-rgw=18.2.1-707-gffcfe7db-1jammy python3-cephfs=18.2.1-707-gffcfe7db-1jammy python3-rbd=18.2.1-707-gffcfe7db-1jammy libcephfs2=18.2.1-707-gffcfe7db-1jammy libcephfs-dev=18.2.1-707-gffcfe7db-1jammy librados2=18.2.1-707-gffcfe7db-1jammy librbd1=18.2.1-707-gffcfe7db-1jammy rbd-fuse=18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:49:17.675 INFO:teuthology.orchestra.run.smithi145.stdout:Reading package lists... 2024-03-07T02:49:17.809 INFO:teuthology.orchestra.run.smithi189.stdout:Reading package lists... 2024-03-07T02:49:17.825 DEBUG:teuthology.orchestra.run.smithi189:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=18.2.1-707-gffcfe7db-1jammy cephadm=18.2.1-707-gffcfe7db-1jammy ceph-mds=18.2.1-707-gffcfe7db-1jammy ceph-mgr=18.2.1-707-gffcfe7db-1jammy ceph-common=18.2.1-707-gffcfe7db-1jammy ceph-fuse=18.2.1-707-gffcfe7db-1jammy ceph-test=18.2.1-707-gffcfe7db-1jammy radosgw=18.2.1-707-gffcfe7db-1jammy python3-rados=18.2.1-707-gffcfe7db-1jammy python3-rgw=18.2.1-707-gffcfe7db-1jammy python3-cephfs=18.2.1-707-gffcfe7db-1jammy python3-rbd=18.2.1-707-gffcfe7db-1jammy libcephfs2=18.2.1-707-gffcfe7db-1jammy libcephfs-dev=18.2.1-707-gffcfe7db-1jammy librados2=18.2.1-707-gffcfe7db-1jammy librbd1=18.2.1-707-gffcfe7db-1jammy rbd-fuse=18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:49:17.868 INFO:teuthology.orchestra.run.smithi145.stdout:Building dependency tree... 2024-03-07T02:49:17.868 INFO:teuthology.orchestra.run.smithi145.stdout:Reading state information... 2024-03-07T02:49:17.889 INFO:teuthology.orchestra.run.smithi189.stdout:Reading package lists... 2024-03-07T02:49:18.050 INFO:teuthology.orchestra.run.smithi145.stdout:The following packages were automatically installed and are no longer required: 2024-03-07T02:49:18.050 INFO:teuthology.orchestra.run.smithi145.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-03-07T02:49:18.051 INFO:teuthology.orchestra.run.smithi145.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-03-07T02:49:18.051 INFO:teuthology.orchestra.run.smithi145.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-03-07T02:49:18.051 INFO:teuthology.orchestra.run.smithi145.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-03-07T02:49:18.051 INFO:teuthology.orchestra.run.smithi145.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-03-07T02:49:18.051 INFO:teuthology.orchestra.run.smithi145.stdout:Use 'sudo apt autoremove' to remove them. 2024-03-07T02:49:18.052 INFO:teuthology.orchestra.run.smithi145.stdout:The following additional packages will be installed: 2024-03-07T02:49:18.052 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-03-07T02:49:18.052 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-03-07T02:49:18.052 INFO:teuthology.orchestra.run.smithi145.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-03-07T02:49:18.052 INFO:teuthology.orchestra.run.smithi145.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-03-07T02:49:18.053 INFO:teuthology.orchestra.run.smithi145.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-03-07T02:49:18.053 INFO:teuthology.orchestra.run.smithi145.stdout: python-asyncssh-doc python-babel-localedata python-pastedeploy-tpl 2024-03-07T02:49:18.053 INFO:teuthology.orchestra.run.smithi145.stdout: python3-asyncssh python3-babel python3-bcrypt python3-cachetools 2024-03-07T02:49:18.053 INFO:teuthology.orchestra.run.smithi145.stdout: python3-ceph-argparse python3-ceph-common python3-certifi python3-cheroot 2024-03-07T02:49:18.053 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-03-07T02:49:18.053 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-03-07T02:49:18.054 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-03-07T02:49:18.054 INFO:teuthology.orchestra.run.smithi145.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-03-07T02:49:18.054 INFO:teuthology.orchestra.run.smithi145.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-03-07T02:49:18.054 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-03-07T02:49:18.054 INFO:teuthology.orchestra.run.smithi145.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-03-07T02:49:18.054 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pygments python3-pyinotify python3-pytest python3-repoze.lru 2024-03-07T02:49:18.054 INFO:teuthology.orchestra.run.smithi145.stdout: python3-requests python3-requests-oauthlib python3-routes python3-rsa 2024-03-07T02:49:18.054 INFO:teuthology.orchestra.run.smithi145.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-03-07T02:49:18.054 INFO:teuthology.orchestra.run.smithi145.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-03-07T02:49:18.054 INFO:teuthology.orchestra.run.smithi145.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-03-07T02:49:18.054 INFO:teuthology.orchestra.run.smithi145.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-03-07T02:49:18.054 INFO:teuthology.orchestra.run.smithi145.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-03-07T02:49:18.055 INFO:teuthology.orchestra.run.smithi145.stdout:Suggested packages: 2024-03-07T02:49:18.055 INFO:teuthology.orchestra.run.smithi145.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-03-07T02:49:18.055 INFO:teuthology.orchestra.run.smithi145.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-03-07T02:49:18.055 INFO:teuthology.orchestra.run.smithi145.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-03-07T02:49:18.055 INFO:teuthology.orchestra.run.smithi145.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-03-07T02:49:18.055 INFO:teuthology.orchestra.run.smithi145.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-03-07T02:49:18.055 INFO:teuthology.orchestra.run.smithi145.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-03-07T02:49:18.055 INFO:teuthology.orchestra.run.smithi145.stdout: python3-watchdog 2024-03-07T02:49:18.055 INFO:teuthology.orchestra.run.smithi145.stdout:Recommended packages: 2024-03-07T02:49:18.055 INFO:teuthology.orchestra.run.smithi145.stdout: btrfs-tools 2024-03-07T02:49:18.090 INFO:teuthology.orchestra.run.smithi189.stdout:Building dependency tree... 2024-03-07T02:49:18.091 INFO:teuthology.orchestra.run.smithi189.stdout:Reading state information... 2024-03-07T02:49:18.154 INFO:teuthology.orchestra.run.smithi145.stdout:The following NEW packages will be installed: 2024-03-07T02:49:18.154 INFO:teuthology.orchestra.run.smithi145.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-03-07T02:49:18.154 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-03-07T02:49:18.154 INFO:teuthology.orchestra.run.smithi145.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-03-07T02:49:18.154 INFO:teuthology.orchestra.run.smithi145.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-03-07T02:49:18.154 INFO:teuthology.orchestra.run.smithi145.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-03-07T02:49:18.155 INFO:teuthology.orchestra.run.smithi145.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-03-07T02:49:18.155 INFO:teuthology.orchestra.run.smithi145.stdout: libthrift-0.16.0 nvme-cli python-asyncssh-doc python-babel-localedata 2024-03-07T02:49:18.155 INFO:teuthology.orchestra.run.smithi145.stdout: python-pastedeploy-tpl python3-asyncssh python3-babel python3-bcrypt 2024-03-07T02:49:18.156 INFO:teuthology.orchestra.run.smithi145.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2024-03-07T02:49:18.156 INFO:teuthology.orchestra.run.smithi145.stdout: python3-certifi python3-cheroot python3-cherrypy3 python3-google-auth 2024-03-07T02:49:18.156 INFO:teuthology.orchestra.run.smithi145.stdout: python3-idna python3-iniconfig python3-jaraco.classes 2024-03-07T02:49:18.156 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2024-03-07T02:49:18.156 INFO:teuthology.orchestra.run.smithi145.stdout: python3-jinja2 python3-joblib python3-kubernetes python3-logutils 2024-03-07T02:49:18.156 INFO:teuthology.orchestra.run.smithi145.stdout: python3-mako python3-markupsafe python3-natsort python3-openssl 2024-03-07T02:49:18.156 INFO:teuthology.orchestra.run.smithi145.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2024-03-07T02:49:18.156 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2024-03-07T02:49:18.156 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pyasn1 python3-pyasn1-modules python3-pygments python3-pyinotify 2024-03-07T02:49:18.156 INFO:teuthology.orchestra.run.smithi145.stdout: python3-pytest python3-rados python3-rbd python3-repoze.lru python3-requests 2024-03-07T02:49:18.156 INFO:teuthology.orchestra.run.smithi145.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2024-03-07T02:49:18.156 INFO:teuthology.orchestra.run.smithi145.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-03-07T02:49:18.156 INFO:teuthology.orchestra.run.smithi145.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-03-07T02:49:18.157 INFO:teuthology.orchestra.run.smithi145.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-03-07T02:49:18.157 INFO:teuthology.orchestra.run.smithi145.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-03-07T02:49:18.157 INFO:teuthology.orchestra.run.smithi145.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2024-03-07T02:49:18.157 INFO:teuthology.orchestra.run.smithi145.stdout: socat xmlstarlet 2024-03-07T02:49:18.266 INFO:teuthology.orchestra.run.smithi189.stdout:The following packages were automatically installed and are no longer required: 2024-03-07T02:49:18.266 INFO:teuthology.orchestra.run.smithi189.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-03-07T02:49:18.266 INFO:teuthology.orchestra.run.smithi189.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-03-07T02:49:18.266 INFO:teuthology.orchestra.run.smithi189.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-03-07T02:49:18.266 INFO:teuthology.orchestra.run.smithi189.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-03-07T02:49:18.267 INFO:teuthology.orchestra.run.smithi189.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-03-07T02:49:18.267 INFO:teuthology.orchestra.run.smithi189.stdout:Use 'sudo apt autoremove' to remove them. 2024-03-07T02:49:18.267 INFO:teuthology.orchestra.run.smithi189.stdout:The following additional packages will be installed: 2024-03-07T02:49:18.268 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-03-07T02:49:18.268 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-03-07T02:49:18.268 INFO:teuthology.orchestra.run.smithi189.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-03-07T02:49:18.268 INFO:teuthology.orchestra.run.smithi189.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: python-asyncssh-doc python-babel-localedata python-pastedeploy-tpl 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: python3-asyncssh python3-babel python3-bcrypt python3-cachetools 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: python3-ceph-argparse python3-ceph-common python3-certifi python3-cheroot 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pygments python3-pyinotify python3-pytest python3-repoze.lru 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: python3-requests python3-requests-oauthlib python3-routes python3-rsa 2024-03-07T02:49:18.269 INFO:teuthology.orchestra.run.smithi189.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-03-07T02:49:18.270 INFO:teuthology.orchestra.run.smithi189.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-03-07T02:49:18.270 INFO:teuthology.orchestra.run.smithi189.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-03-07T02:49:18.270 INFO:teuthology.orchestra.run.smithi189.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-03-07T02:49:18.270 INFO:teuthology.orchestra.run.smithi189.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-03-07T02:49:18.270 INFO:teuthology.orchestra.run.smithi189.stdout:Suggested packages: 2024-03-07T02:49:18.270 INFO:teuthology.orchestra.run.smithi189.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-03-07T02:49:18.270 INFO:teuthology.orchestra.run.smithi189.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-03-07T02:49:18.270 INFO:teuthology.orchestra.run.smithi189.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-03-07T02:49:18.270 INFO:teuthology.orchestra.run.smithi189.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-03-07T02:49:18.270 INFO:teuthology.orchestra.run.smithi189.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-03-07T02:49:18.271 INFO:teuthology.orchestra.run.smithi189.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-03-07T02:49:18.271 INFO:teuthology.orchestra.run.smithi189.stdout: python3-watchdog 2024-03-07T02:49:18.271 INFO:teuthology.orchestra.run.smithi189.stdout:Recommended packages: 2024-03-07T02:49:18.271 INFO:teuthology.orchestra.run.smithi189.stdout: btrfs-tools 2024-03-07T02:49:18.343 INFO:teuthology.orchestra.run.smithi145.stdout:0 upgraded, 109 newly installed, 0 to remove and 275 not upgraded. 2024-03-07T02:49:18.343 INFO:teuthology.orchestra.run.smithi145.stdout:Need to get 181 MB of archives. 2024-03-07T02:49:18.343 INFO:teuthology.orchestra.run.smithi145.stdout:After this operation, 811 MB of additional disk space will be used. 2024-03-07T02:49:18.343 INFO:teuthology.orchestra.run.smithi145.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-03-07T02:49:18.372 INFO:teuthology.orchestra.run.smithi189.stdout:The following NEW packages will be installed: 2024-03-07T02:49:18.372 INFO:teuthology.orchestra.run.smithi189.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-03-07T02:49:18.372 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-03-07T02:49:18.372 INFO:teuthology.orchestra.run.smithi189.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-03-07T02:49:18.373 INFO:teuthology.orchestra.run.smithi145.stdout:Get:2 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 18.2.1-707-gffcfe7db-1jammy [3,418 kB] 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: libthrift-0.16.0 nvme-cli python-asyncssh-doc python-babel-localedata 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: python-pastedeploy-tpl python3-asyncssh python3-babel python3-bcrypt 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: python3-certifi python3-cheroot python3-cherrypy3 python3-google-auth 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: python3-idna python3-iniconfig python3-jaraco.classes 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: python3-jinja2 python3-joblib python3-kubernetes python3-logutils 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: python3-mako python3-markupsafe python3-natsort python3-openssl 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2024-03-07T02:49:18.374 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pyasn1 python3-pyasn1-modules python3-pygments python3-pyinotify 2024-03-07T02:49:18.375 INFO:teuthology.orchestra.run.smithi189.stdout: python3-pytest python3-rados python3-rbd python3-repoze.lru python3-requests 2024-03-07T02:49:18.375 INFO:teuthology.orchestra.run.smithi189.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2024-03-07T02:49:18.375 INFO:teuthology.orchestra.run.smithi189.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-03-07T02:49:18.375 INFO:teuthology.orchestra.run.smithi189.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-03-07T02:49:18.375 INFO:teuthology.orchestra.run.smithi189.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-03-07T02:49:18.375 INFO:teuthology.orchestra.run.smithi189.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-03-07T02:49:18.375 INFO:teuthology.orchestra.run.smithi189.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2024-03-07T02:49:18.375 INFO:teuthology.orchestra.run.smithi189.stdout: socat xmlstarlet 2024-03-07T02:49:18.438 INFO:teuthology.orchestra.run.smithi189.stdout:0 upgraded, 109 newly installed, 0 to remove and 275 not upgraded. 2024-03-07T02:49:18.438 INFO:teuthology.orchestra.run.smithi189.stdout:Need to get 181 MB of archives. 2024-03-07T02:49:18.439 INFO:teuthology.orchestra.run.smithi189.stdout:After this operation, 811 MB of additional disk space will be used. 2024-03-07T02:49:18.439 INFO:teuthology.orchestra.run.smithi189.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-03-07T02:49:18.504 INFO:teuthology.orchestra.run.smithi189.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-03-07T02:49:18.515 INFO:teuthology.orchestra.run.smithi189.stdout:Get:3 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 18.2.1-707-gffcfe7db-1jammy [3,418 kB] 2024-03-07T02:49:18.546 INFO:teuthology.orchestra.run.smithi189.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-03-07T02:49:18.549 INFO:teuthology.orchestra.run.smithi189.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-03-07T02:49:18.566 INFO:teuthology.orchestra.run.smithi189.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-03-07T02:49:18.622 INFO:teuthology.orchestra.run.smithi189.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-03-07T02:49:18.626 INFO:teuthology.orchestra.run.smithi189.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-03-07T02:49:18.626 INFO:teuthology.orchestra.run.smithi145.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-03-07T02:49:18.637 INFO:teuthology.orchestra.run.smithi189.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-03-07T02:49:18.641 INFO:teuthology.orchestra.run.smithi189.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-03-07T02:49:18.642 INFO:teuthology.orchestra.run.smithi189.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-03-07T02:49:18.643 INFO:teuthology.orchestra.run.smithi189.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-03-07T02:49:18.651 INFO:teuthology.orchestra.run.smithi189.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-03-07T02:49:18.652 INFO:teuthology.orchestra.run.smithi189.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-03-07T02:49:18.652 INFO:teuthology.orchestra.run.smithi189.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-03-07T02:49:18.653 INFO:teuthology.orchestra.run.smithi189.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-03-07T02:49:18.665 INFO:teuthology.orchestra.run.smithi189.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-03-07T02:49:18.665 INFO:teuthology.orchestra.run.smithi189.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-03-07T02:49:18.666 INFO:teuthology.orchestra.run.smithi189.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-03-07T02:49:18.666 INFO:teuthology.orchestra.run.smithi189.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-03-07T02:49:18.685 INFO:teuthology.orchestra.run.smithi189.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-03-07T02:49:18.685 INFO:teuthology.orchestra.run.smithi189.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-03-07T02:49:18.687 INFO:teuthology.orchestra.run.smithi189.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-03-07T02:49:18.688 INFO:teuthology.orchestra.run.smithi189.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-03-07T02:49:18.688 INFO:teuthology.orchestra.run.smithi189.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-03-07T02:49:18.689 INFO:teuthology.orchestra.run.smithi189.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-03-07T02:49:18.704 INFO:teuthology.orchestra.run.smithi189.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-03-07T02:49:18.704 INFO:teuthology.orchestra.run.smithi189.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-03-07T02:49:18.705 INFO:teuthology.orchestra.run.smithi189.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webob all 1:1.8.6-1.1 [85.2 kB] 2024-03-07T02:49:18.706 INFO:teuthology.orchestra.run.smithi189.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-03-07T02:49:18.723 INFO:teuthology.orchestra.run.smithi189.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-03-07T02:49:18.724 INFO:teuthology.orchestra.run.smithi189.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-03-07T02:49:18.729 INFO:teuthology.orchestra.run.smithi189.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-03-07T02:49:18.729 INFO:teuthology.orchestra.run.smithi189.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-03-07T02:49:18.730 INFO:teuthology.orchestra.run.smithi189.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-03-07T02:49:18.731 INFO:teuthology.orchestra.run.smithi189.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-03-07T02:49:18.743 INFO:teuthology.orchestra.run.smithi189.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-03-07T02:49:18.745 INFO:teuthology.orchestra.run.smithi189.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-idna all 3.3-1 [49.3 kB] 2024-03-07T02:49:18.745 INFO:teuthology.orchestra.run.smithi189.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-03-07T02:49:18.747 INFO:teuthology.orchestra.run.smithi189.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-03-07T02:49:18.762 INFO:teuthology.orchestra.run.smithi189.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.1 [181 kB] 2024-03-07T02:49:18.765 INFO:teuthology.orchestra.run.smithi189.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-03-07T02:49:18.766 INFO:teuthology.orchestra.run.smithi189.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-asyncssh all 2.5.0-1 [185 kB] 2024-03-07T02:49:18.766 INFO:teuthology.orchestra.run.smithi189.stdout:Get:44 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 18.2.1-707-gffcfe7db-1jammy [3,105 kB] 2024-03-07T02:49:18.768 INFO:teuthology.orchestra.run.smithi189.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-03-07T02:49:18.779 INFO:teuthology.orchestra.run.smithi145.stdout:Get:4 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 18.2.1-707-gffcfe7db-1jammy [3,105 kB] 2024-03-07T02:49:18.797 INFO:teuthology.orchestra.run.smithi145.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-03-07T02:49:18.812 INFO:teuthology.orchestra.run.smithi145.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-03-07T02:49:18.831 INFO:teuthology.orchestra.run.smithi189.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-03-07T02:49:18.832 INFO:teuthology.orchestra.run.smithi189.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.1 [108 kB] 2024-03-07T02:49:18.833 INFO:teuthology.orchestra.run.smithi189.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-03-07T02:49:18.834 INFO:teuthology.orchestra.run.smithi189.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-03-07T02:49:18.834 INFO:teuthology.orchestra.run.smithi189.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-03-07T02:49:18.836 INFO:teuthology.orchestra.run.smithi189.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-03-07T02:49:18.836 INFO:teuthology.orchestra.run.smithi189.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-03-07T02:49:18.836 INFO:teuthology.orchestra.run.smithi189.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-03-07T02:49:18.839 INFO:teuthology.orchestra.run.smithi189.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-03-07T02:49:18.854 INFO:teuthology.orchestra.run.smithi189.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-03-07T02:49:18.855 INFO:teuthology.orchestra.run.smithi189.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-03-07T02:49:18.876 INFO:teuthology.orchestra.run.smithi189.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-03-07T02:49:18.881 INFO:teuthology.orchestra.run.smithi189.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-03-07T02:49:18.881 INFO:teuthology.orchestra.run.smithi189.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-03-07T02:49:18.885 INFO:teuthology.orchestra.run.smithi145.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-03-07T02:49:18.900 INFO:teuthology.orchestra.run.smithi189.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-03-07T02:49:18.900 INFO:teuthology.orchestra.run.smithi189.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-03-07T02:49:18.902 INFO:teuthology.orchestra.run.smithi189.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-03-07T02:49:18.902 INFO:teuthology.orchestra.run.smithi189.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-03-07T02:49:18.903 INFO:teuthology.orchestra.run.smithi189.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-03-07T02:49:18.906 INFO:teuthology.orchestra.run.smithi189.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-03-07T02:49:18.906 INFO:teuthology.orchestra.run.smithi189.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-03-07T02:49:18.907 INFO:teuthology.orchestra.run.smithi189.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-03-07T02:49:18.914 INFO:teuthology.orchestra.run.smithi189.stdout:Get:68 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 18.2.1-707-gffcfe7db-1jammy [672 kB] 2024-03-07T02:49:18.926 INFO:teuthology.orchestra.run.smithi189.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-03-07T02:49:18.928 INFO:teuthology.orchestra.run.smithi189.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-03-07T02:49:18.930 INFO:teuthology.orchestra.run.smithi189.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-03-07T02:49:18.930 INFO:teuthology.orchestra.run.smithi189.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-03-07T02:49:18.934 INFO:teuthology.orchestra.run.smithi189.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-03-07T02:49:18.937 INFO:teuthology.orchestra.run.smithi189.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.1 [474 kB] 2024-03-07T02:49:18.942 INFO:teuthology.orchestra.run.smithi189.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python-asyncssh-doc all 2.5.0-1 [318 kB] 2024-03-07T02:49:18.951 INFO:teuthology.orchestra.run.smithi189.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-03-07T02:49:18.951 INFO:teuthology.orchestra.run.smithi189.stdout:Get:77 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 18.2.1-707-gffcfe7db-1jammy [356 kB] 2024-03-07T02:49:18.952 INFO:teuthology.orchestra.run.smithi189.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-03-07T02:49:18.954 INFO:teuthology.orchestra.run.smithi189.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-03-07T02:49:18.970 INFO:teuthology.orchestra.run.smithi189.stdout:Get:80 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 18.2.1-707-gffcfe7db-1jammy [32.8 kB] 2024-03-07T02:49:18.970 INFO:teuthology.orchestra.run.smithi189.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-03-07T02:49:18.971 INFO:teuthology.orchestra.run.smithi189.stdout:Get:82 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 18.2.1-707-gffcfe7db-1jammy [180 kB] 2024-03-07T02:49:18.971 INFO:teuthology.orchestra.run.smithi189.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-03-07T02:49:18.971 INFO:teuthology.orchestra.run.smithi189.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-03-07T02:49:18.977 INFO:teuthology.orchestra.run.smithi189.stdout:Get:85 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 18.2.1-707-gffcfe7db-1jammy [62.1 kB] 2024-03-07T02:49:18.979 INFO:teuthology.orchestra.run.smithi189.stdout:Get:86 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 18.2.1-707-gffcfe7db-1jammy [330 kB] 2024-03-07T02:49:18.998 INFO:teuthology.orchestra.run.smithi189.stdout:Get:87 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 18.2.1-707-gffcfe7db-1jammy [5,887 kB] 2024-03-07T02:49:19.024 INFO:teuthology.orchestra.run.smithi145.stdout:Get:8 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 18.2.1-707-gffcfe7db-1jammy [672 kB] 2024-03-07T02:49:19.078 INFO:teuthology.orchestra.run.smithi145.stdout:Get:9 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 18.2.1-707-gffcfe7db-1jammy [356 kB] 2024-03-07T02:49:19.080 INFO:teuthology.orchestra.run.smithi145.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-03-07T02:49:19.090 INFO:teuthology.orchestra.run.smithi145.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-03-07T02:49:19.111 INFO:teuthology.orchestra.run.smithi145.stdout:Get:12 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 18.2.1-707-gffcfe7db-1jammy [32.8 kB] 2024-03-07T02:49:19.115 INFO:teuthology.orchestra.run.smithi145.stdout:Get:13 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 18.2.1-707-gffcfe7db-1jammy [180 kB] 2024-03-07T02:49:19.118 INFO:teuthology.orchestra.run.smithi145.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-03-07T02:49:19.127 INFO:teuthology.orchestra.run.smithi145.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-03-07T02:49:19.128 INFO:teuthology.orchestra.run.smithi145.stdout:Get:16 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 18.2.1-707-gffcfe7db-1jammy [62.1 kB] 2024-03-07T02:49:19.128 INFO:teuthology.orchestra.run.smithi145.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-03-07T02:49:19.129 INFO:teuthology.orchestra.run.smithi145.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-03-07T02:49:19.132 INFO:teuthology.orchestra.run.smithi145.stdout:Get:19 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 18.2.1-707-gffcfe7db-1jammy [330 kB] 2024-03-07T02:49:19.160 INFO:teuthology.orchestra.run.smithi145.stdout:Get:20 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 18.2.1-707-gffcfe7db-1jammy [5,887 kB] 2024-03-07T02:49:19.310 INFO:teuthology.orchestra.run.smithi145.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-03-07T02:49:19.324 INFO:teuthology.orchestra.run.smithi189.stdout:Get:88 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 18.2.1-707-gffcfe7db-1jammy [112 kB] 2024-03-07T02:49:19.329 INFO:teuthology.orchestra.run.smithi189.stdout:Get:89 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 18.2.1-707-gffcfe7db-1jammy [441 kB] 2024-03-07T02:49:19.353 INFO:teuthology.orchestra.run.smithi189.stdout:Get:90 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 18.2.1-707-gffcfe7db-1jammy [22.0 MB] 2024-03-07T02:49:19.525 INFO:teuthology.orchestra.run.smithi145.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-03-07T02:49:19.539 INFO:teuthology.orchestra.run.smithi145.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-03-07T02:49:19.642 INFO:teuthology.orchestra.run.smithi145.stdout:Get:24 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 18.2.1-707-gffcfe7db-1jammy [112 kB] 2024-03-07T02:49:19.651 INFO:teuthology.orchestra.run.smithi145.stdout:Get:25 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 18.2.1-707-gffcfe7db-1jammy [441 kB] 2024-03-07T02:49:19.658 INFO:teuthology.orchestra.run.smithi145.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-03-07T02:49:19.667 INFO:teuthology.orchestra.run.smithi145.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-03-07T02:49:19.678 INFO:teuthology.orchestra.run.smithi145.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-03-07T02:49:19.694 INFO:teuthology.orchestra.run.smithi145.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-03-07T02:49:19.707 INFO:teuthology.orchestra.run.smithi145.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-03-07T02:49:19.712 INFO:teuthology.orchestra.run.smithi145.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-03-07T02:49:19.717 INFO:teuthology.orchestra.run.smithi145.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-03-07T02:49:19.788 INFO:teuthology.orchestra.run.smithi145.stdout:Get:33 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 18.2.1-707-gffcfe7db-1jammy [22.0 MB] 2024-03-07T02:49:19.820 INFO:teuthology.orchestra.run.smithi145.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-03-07T02:49:19.995 INFO:teuthology.orchestra.run.smithi145.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-03-07T02:49:20.164 INFO:teuthology.orchestra.run.smithi145.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-03-07T02:49:20.190 INFO:teuthology.orchestra.run.smithi145.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-03-07T02:49:20.321 INFO:teuthology.orchestra.run.smithi145.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-03-07T02:49:20.339 INFO:teuthology.orchestra.run.smithi145.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-03-07T02:49:20.346 INFO:teuthology.orchestra.run.smithi145.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webob all 1:1.8.6-1.1 [85.2 kB] 2024-03-07T02:49:20.417 INFO:teuthology.orchestra.run.smithi145.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-03-07T02:49:20.444 INFO:teuthology.orchestra.run.smithi145.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-03-07T02:49:20.452 INFO:teuthology.orchestra.run.smithi145.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-03-07T02:49:20.575 INFO:teuthology.orchestra.run.smithi145.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-03-07T02:49:20.575 INFO:teuthology.orchestra.run.smithi145.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-03-07T02:49:20.613 INFO:teuthology.orchestra.run.smithi189.stdout:Get:91 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 18.2.1-707-gffcfe7db-1jammy [6,849 kB] 2024-03-07T02:49:20.744 INFO:teuthology.orchestra.run.smithi145.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-03-07T02:49:20.893 INFO:teuthology.orchestra.run.smithi189.stdout:Get:92 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 18.2.1-707-gffcfe7db-1jammy [240 kB] 2024-03-07T02:49:20.902 INFO:teuthology.orchestra.run.smithi189.stdout:Get:93 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 18.2.1-707-gffcfe7db-1jammy [130 kB] 2024-03-07T02:49:20.909 INFO:teuthology.orchestra.run.smithi189.stdout:Get:94 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 18.2.1-707-gffcfe7db-1jammy [1,484 kB] 2024-03-07T02:49:20.945 INFO:teuthology.orchestra.run.smithi145.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-03-07T02:49:20.989 INFO:teuthology.orchestra.run.smithi189.stdout:Get:95 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 18.2.1-707-gffcfe7db-1jammy [9,906 kB] 2024-03-07T02:49:21.107 INFO:teuthology.orchestra.run.smithi145.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-03-07T02:49:21.211 INFO:teuthology.orchestra.run.smithi145.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-idna all 3.3-1 [49.3 kB] 2024-03-07T02:49:21.233 INFO:teuthology.orchestra.run.smithi145.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-03-07T02:49:21.263 INFO:teuthology.orchestra.run.smithi145.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-03-07T02:49:21.276 INFO:teuthology.orchestra.run.smithi145.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.1 [181 kB] 2024-03-07T02:49:21.317 INFO:teuthology.orchestra.run.smithi145.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-03-07T02:49:21.325 INFO:teuthology.orchestra.run.smithi145.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-asyncssh all 2.5.0-1 [185 kB] 2024-03-07T02:49:21.356 INFO:teuthology.orchestra.run.smithi145.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-03-07T02:49:21.522 INFO:teuthology.orchestra.run.smithi189.stdout:Get:96 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 18.2.1-707-gffcfe7db-1jammy [31.6 MB] 2024-03-07T02:49:21.583 INFO:teuthology.orchestra.run.smithi145.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-03-07T02:49:21.669 INFO:teuthology.orchestra.run.smithi145.stdout:Get:57 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 18.2.1-707-gffcfe7db-1jammy [6,849 kB] 2024-03-07T02:49:21.831 INFO:teuthology.orchestra.run.smithi145.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.1 [108 kB] 2024-03-07T02:49:22.153 INFO:teuthology.orchestra.run.smithi145.stdout:Get:59 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 18.2.1-707-gffcfe7db-1jammy [240 kB] 2024-03-07T02:49:22.171 INFO:teuthology.orchestra.run.smithi145.stdout:Get:60 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 18.2.1-707-gffcfe7db-1jammy [130 kB] 2024-03-07T02:49:22.181 INFO:teuthology.orchestra.run.smithi145.stdout:Get:61 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 18.2.1-707-gffcfe7db-1jammy [1,484 kB] 2024-03-07T02:49:22.189 INFO:teuthology.orchestra.run.smithi145.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-03-07T02:49:22.207 INFO:teuthology.orchestra.run.smithi145.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-03-07T02:49:22.218 INFO:teuthology.orchestra.run.smithi145.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-03-07T02:49:22.272 INFO:teuthology.orchestra.run.smithi145.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-03-07T02:49:22.276 INFO:teuthology.orchestra.run.smithi145.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-03-07T02:49:22.285 INFO:teuthology.orchestra.run.smithi145.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-03-07T02:49:22.306 INFO:teuthology.orchestra.run.smithi145.stdout:Get:68 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 18.2.1-707-gffcfe7db-1jammy [9,906 kB] 2024-03-07T02:49:22.372 INFO:teuthology.orchestra.run.smithi145.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-03-07T02:49:22.375 INFO:teuthology.orchestra.run.smithi145.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-03-07T02:49:22.396 INFO:teuthology.orchestra.run.smithi145.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-03-07T02:49:22.589 INFO:teuthology.orchestra.run.smithi145.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-03-07T02:49:22.760 INFO:teuthology.orchestra.run.smithi145.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-03-07T02:49:22.933 INFO:teuthology.orchestra.run.smithi145.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-03-07T02:49:23.188 INFO:teuthology.orchestra.run.smithi145.stdout:Get:75 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 18.2.1-707-gffcfe7db-1jammy [31.6 MB] 2024-03-07T02:49:23.316 INFO:teuthology.orchestra.run.smithi189.stdout:Get:97 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 18.2.1-707-gffcfe7db-1jammy [14.1 kB] 2024-03-07T02:49:23.317 INFO:teuthology.orchestra.run.smithi189.stdout:Get:98 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 18.2.1-707-gffcfe7db-1jammy [824 kB] 2024-03-07T02:49:23.333 INFO:teuthology.orchestra.run.smithi189.stdout:Get:99 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 18.2.1-707-gffcfe7db-1jammy [2,175 kB] 2024-03-07T02:49:23.384 INFO:teuthology.orchestra.run.smithi189.stdout:Get:100 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 18.2.1-707-gffcfe7db-1jammy [237 kB] 2024-03-07T02:49:23.391 INFO:teuthology.orchestra.run.smithi189.stdout:Get:101 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 18.2.1-707-gffcfe7db-1jammy [143 kB] 2024-03-07T02:49:23.438 INFO:teuthology.orchestra.run.smithi145.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-03-07T02:49:23.439 INFO:teuthology.orchestra.run.smithi145.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-03-07T02:49:23.442 INFO:teuthology.orchestra.run.smithi145.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-03-07T02:49:23.444 INFO:teuthology.orchestra.run.smithi145.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-03-07T02:49:23.445 INFO:teuthology.orchestra.run.smithi145.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-03-07T02:49:23.448 INFO:teuthology.orchestra.run.smithi145.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-03-07T02:49:23.449 INFO:teuthology.orchestra.run.smithi145.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-03-07T02:49:23.451 INFO:teuthology.orchestra.run.smithi145.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-03-07T02:49:23.469 INFO:teuthology.orchestra.run.smithi145.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-03-07T02:49:23.479 INFO:teuthology.orchestra.run.smithi189.stdout:Get:102 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 18.2.1-707-gffcfe7db-1jammy [1,880 kB] 2024-03-07T02:49:23.514 INFO:teuthology.orchestra.run.smithi189.stdout:Get:103 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 18.2.1-707-gffcfe7db-1jammy [8,625 kB] 2024-03-07T02:49:23.641 INFO:teuthology.orchestra.run.smithi145.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-03-07T02:49:24.021 INFO:teuthology.orchestra.run.smithi145.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-03-07T02:49:24.044 INFO:teuthology.orchestra.run.smithi145.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-03-07T02:49:24.049 INFO:teuthology.orchestra.run.smithi189.stdout:Get:104 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 18.2.1-707-gffcfe7db-1jammy [14.3 kB] 2024-03-07T02:49:24.050 INFO:teuthology.orchestra.run.smithi189.stdout:Get:105 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 18.2.1-707-gffcfe7db-1jammy [46.6 MB] 2024-03-07T02:49:24.196 INFO:teuthology.orchestra.run.smithi145.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-03-07T02:49:24.246 INFO:teuthology.orchestra.run.smithi145.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.1 [474 kB] 2024-03-07T02:49:24.308 INFO:teuthology.orchestra.run.smithi145.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python-asyncssh-doc all 2.5.0-1 [318 kB] 2024-03-07T02:49:24.334 INFO:teuthology.orchestra.run.smithi145.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-03-07T02:49:24.338 INFO:teuthology.orchestra.run.smithi145.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-03-07T02:49:24.350 INFO:teuthology.orchestra.run.smithi145.stdout:Get:93 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-03-07T02:49:24.393 INFO:teuthology.orchestra.run.smithi145.stdout:Get:94 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-03-07T02:49:24.394 INFO:teuthology.orchestra.run.smithi145.stdout:Get:95 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-03-07T02:49:24.560 INFO:teuthology.orchestra.run.smithi145.stdout:Get:96 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-03-07T02:49:25.559 INFO:teuthology.orchestra.run.smithi145.stdout:Get:97 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 18.2.1-707-gffcfe7db-1jammy [14.1 kB] 2024-03-07T02:49:25.560 INFO:teuthology.orchestra.run.smithi145.stdout:Get:98 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 18.2.1-707-gffcfe7db-1jammy [824 kB] 2024-03-07T02:49:25.576 INFO:teuthology.orchestra.run.smithi145.stdout:Get:99 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 18.2.1-707-gffcfe7db-1jammy [2,175 kB] 2024-03-07T02:49:25.644 INFO:teuthology.orchestra.run.smithi145.stdout:Get:100 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 18.2.1-707-gffcfe7db-1jammy [237 kB] 2024-03-07T02:49:25.657 INFO:teuthology.orchestra.run.smithi145.stdout:Get:101 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 18.2.1-707-gffcfe7db-1jammy [143 kB] 2024-03-07T02:49:25.669 INFO:teuthology.orchestra.run.smithi145.stdout:Get:102 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 18.2.1-707-gffcfe7db-1jammy [1,880 kB] 2024-03-07T02:49:25.804 INFO:teuthology.orchestra.run.smithi145.stdout:Get:103 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 18.2.1-707-gffcfe7db-1jammy [8,625 kB] 2024-03-07T02:49:26.522 INFO:teuthology.orchestra.run.smithi145.stdout:Get:104 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 18.2.1-707-gffcfe7db-1jammy [14.3 kB] 2024-03-07T02:49:26.523 INFO:teuthology.orchestra.run.smithi145.stdout:Get:105 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 18.2.1-707-gffcfe7db-1jammy [46.6 MB] 2024-03-07T02:49:26.923 INFO:teuthology.orchestra.run.smithi189.stdout:Get:106 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 18.2.1-707-gffcfe7db-1jammy [121 kB] 2024-03-07T02:49:26.925 INFO:teuthology.orchestra.run.smithi189.stdout:Get:107 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 18.2.1-707-gffcfe7db-1jammy [38.3 kB] 2024-03-07T02:49:26.926 INFO:teuthology.orchestra.run.smithi189.stdout:Get:108 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 18.2.1-707-gffcfe7db-1jammy [11.6 MB] 2024-03-07T02:49:27.360 INFO:teuthology.orchestra.run.smithi189.stdout:Get:109 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 18.2.1-707-gffcfe7db-1jammy [93.1 kB] 2024-03-07T02:49:27.906 INFO:teuthology.orchestra.run.smithi189.stdout:Fetched 181 MB in 9s (20.1 MB/s) 2024-03-07T02:49:28.050 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package libfmt8:amd64. 2024-03-07T02:49:28.146 INFO:teuthology.orchestra.run.smithi189.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165994 files and directories currently installed.) 2024-03-07T02:49:28.151 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-03-07T02:49:28.231 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-03-07T02:49:28.551 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-03-07T02:49:28.578 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-03-07T02:49:28.617 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-03-07T02:49:28.929 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-03-07T02:49:28.955 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../002-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-03-07T02:49:28.996 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-03-07T02:49:29.315 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-03-07T02:49:29.341 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../003-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-03-07T02:49:29.373 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-03-07T02:49:29.810 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-03-07T02:49:29.836 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../004-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-03-07T02:49:29.885 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:49:30.288 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-03-07T02:49:30.315 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../005-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-03-07T02:49:30.350 INFO:teuthology.orchestra.run.smithi145.stdout:Get:106 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 18.2.1-707-gffcfe7db-1jammy [121 kB] 2024-03-07T02:49:30.352 INFO:teuthology.orchestra.run.smithi145.stdout:Get:107 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 18.2.1-707-gffcfe7db-1jammy [38.3 kB] 2024-03-07T02:49:30.354 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:49:30.420 INFO:teuthology.orchestra.run.smithi145.stdout:Get:108 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 18.2.1-707-gffcfe7db-1jammy [11.6 MB] 2024-03-07T02:49:30.891 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-03-07T02:49:30.917 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../006-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-03-07T02:49:30.949 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:49:31.302 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-03-07T02:49:31.328 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../007-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-03-07T02:49:31.369 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-03-07T02:49:31.688 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package librados2. 2024-03-07T02:49:31.714 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../008-librados2_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:31.754 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking librados2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:32.135 INFO:teuthology.orchestra.run.smithi145.stdout:Get:109 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 18.2.1-707-gffcfe7db-1jammy [93.1 kB] 2024-03-07T02:49:32.207 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package librbd1. 2024-03-07T02:49:32.233 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../009-librbd1_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:32.273 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking librbd1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:32.692 INFO:teuthology.orchestra.run.smithi145.stdout:Fetched 181 MB in 14s (12.9 MB/s) 2024-03-07T02:49:32.693 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package libcephfs2. 2024-03-07T02:49:32.719 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../010-libcephfs2_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:32.760 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking libcephfs2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:32.831 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package libfmt8:amd64. 2024-03-07T02:49:32.929 INFO:teuthology.orchestra.run.smithi145.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165994 files and directories currently installed.) 2024-03-07T02:49:32.935 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-03-07T02:49:33.021 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-03-07T02:49:33.037 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-rados. 2024-03-07T02:49:33.052 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../011-python3-rados_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:33.095 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-rados (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:33.375 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-03-07T02:49:33.400 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-03-07T02:49:33.414 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-03-07T02:49:33.440 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../012-python3-ceph-argparse_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:49:33.441 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-03-07T02:49:33.481 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-ceph-argparse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:33.733 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-cephfs. 2024-03-07T02:49:33.760 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../013-python3-cephfs_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:33.795 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-03-07T02:49:33.800 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-cephfs (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:33.821 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../002-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-03-07T02:49:33.861 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-03-07T02:49:34.086 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-ceph-common. 2024-03-07T02:49:34.112 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../014-python3-ceph-common_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:49:34.144 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:34.147 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-03-07T02:49:34.173 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../003-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-03-07T02:49:34.205 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-03-07T02:49:34.463 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-wcwidth. 2024-03-07T02:49:34.490 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../015-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-03-07T02:49:34.530 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-03-07T02:49:34.659 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-03-07T02:49:34.686 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../004-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-03-07T02:49:34.734 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:49:34.849 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-prettytable. 2024-03-07T02:49:34.875 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../016-python3-prettytable_2.5.0-2_all.deb ... 2024-03-07T02:49:34.916 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-03-07T02:49:35.143 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-rbd. 2024-03-07T02:49:35.158 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../017-python3-rbd_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:35.171 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-03-07T02:49:35.193 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-rbd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:35.197 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../005-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-03-07T02:49:35.229 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:49:35.538 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-03-07T02:49:35.564 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../018-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-03-07T02:49:35.583 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-03-07T02:49:35.604 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-03-07T02:49:35.609 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../006-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-03-07T02:49:35.649 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:49:35.932 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package librgw2. 2024-03-07T02:49:35.947 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../019-librgw2_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:35.990 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking librgw2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:36.236 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-03-07T02:49:36.262 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../007-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-03-07T02:49:36.302 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-03-07T02:49:36.501 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-rgw. 2024-03-07T02:49:36.516 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../020-python3-rgw_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:36.559 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-rgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:36.588 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package librados2. 2024-03-07T02:49:36.604 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../008-librados2_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:36.638 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking librados2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:36.879 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package liboath0:amd64. 2024-03-07T02:49:36.905 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-03-07T02:49:36.945 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-03-07T02:49:37.100 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package librbd1. 2024-03-07T02:49:37.126 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../009-librbd1_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:37.166 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking librbd1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:37.206 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package libradosstriper1. 2024-03-07T02:49:37.233 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../022-libradosstriper1_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:37.273 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking libradosstriper1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:37.610 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package libcephfs2. 2024-03-07T02:49:37.637 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../010-libcephfs2_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:37.677 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking libcephfs2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:37.701 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-common. 2024-03-07T02:49:37.727 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../023-ceph-common_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:37.768 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:37.954 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-rados. 2024-03-07T02:49:37.970 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../011-python3-rados_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:38.004 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-rados (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:38.332 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-03-07T02:49:38.359 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../012-python3-ceph-argparse_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:49:38.399 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-ceph-argparse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:38.651 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-cephfs. 2024-03-07T02:49:38.667 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../013-python3-cephfs_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:38.709 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-cephfs (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:39.015 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-base. 2024-03-07T02:49:39.029 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-ceph-common. 2024-03-07T02:49:39.041 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../024-ceph-base_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:39.055 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../014-python3-ceph-common_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:49:39.090 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-base (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:39.095 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:39.449 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-wcwidth. 2024-03-07T02:49:39.475 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../015-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-03-07T02:49:39.515 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-03-07T02:49:39.793 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-prettytable. 2024-03-07T02:49:39.794 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-03-07T02:49:39.820 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../016-python3-prettytable_2.5.0-2_all.deb ... 2024-03-07T02:49:39.820 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../025-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-03-07T02:49:39.860 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-03-07T02:49:39.860 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-03-07T02:49:40.129 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-rbd. 2024-03-07T02:49:40.145 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../017-python3-rbd_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:40.146 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-cheroot. 2024-03-07T02:49:40.173 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../026-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-03-07T02:49:40.187 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-rbd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:40.213 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-03-07T02:49:40.540 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-03-07T02:49:40.541 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-03-07T02:49:40.567 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../018-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-03-07T02:49:40.567 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../027-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-03-07T02:49:40.607 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-03-07T02:49:40.615 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-03-07T02:49:40.860 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-jaraco.text. 2024-03-07T02:49:40.875 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../028-python3-jaraco.text_3.6.0-2_all.deb ... 2024-03-07T02:49:40.910 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-03-07T02:49:40.943 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package librgw2. 2024-03-07T02:49:40.970 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../019-librgw2_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:41.009 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking librgw2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:41.229 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-03-07T02:49:41.256 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../029-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-03-07T02:49:41.296 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-03-07T02:49:41.637 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-rgw. 2024-03-07T02:49:41.652 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../020-python3-rgw_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:41.687 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-rgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:41.823 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-tempora. 2024-03-07T02:49:41.850 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../030-python3-tempora_4.1.2-1_all.deb ... 2024-03-07T02:49:41.890 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-03-07T02:49:42.023 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package liboath0:amd64. 2024-03-07T02:49:42.050 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-03-07T02:49:42.090 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-03-07T02:49:42.170 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-portend. 2024-03-07T02:49:42.196 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../031-python3-portend_3.0.0-1_all.deb ... 2024-03-07T02:49:42.236 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-03-07T02:49:42.410 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package libradosstriper1. 2024-03-07T02:49:42.436 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../022-libradosstriper1_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:42.468 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking libradosstriper1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:42.539 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-03-07T02:49:42.565 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../032-python3-zc.lockfile_2.0-1_all.deb ... 2024-03-07T02:49:42.606 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-03-07T02:49:42.863 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-common. 2024-03-07T02:49:42.889 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../023-ceph-common_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:42.909 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-cherrypy3. 2024-03-07T02:49:42.929 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:42.935 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../033-python3-cherrypy3_18.6.1-4_all.deb ... 2024-03-07T02:49:42.967 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-03-07T02:49:43.321 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-natsort. 2024-03-07T02:49:43.347 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../034-python3-natsort_8.0.2-1_all.deb ... 2024-03-07T02:49:43.387 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-03-07T02:49:43.699 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-logutils. 2024-03-07T02:49:43.726 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../035-python3-logutils_0.3.3-8_all.deb ... 2024-03-07T02:49:43.765 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-03-07T02:49:44.043 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-markupsafe. 2024-03-07T02:49:44.070 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../036-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-03-07T02:49:44.110 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-03-07T02:49:44.229 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-base. 2024-03-07T02:49:44.255 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../024-ceph-base_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:44.313 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-base (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:44.362 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-mako. 2024-03-07T02:49:44.389 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../037-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-03-07T02:49:44.429 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-03-07T02:49:44.758 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-simplegeneric. 2024-03-07T02:49:44.784 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../038-python3-simplegeneric_0.8.1-3_all.deb ... 2024-03-07T02:49:44.824 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-03-07T02:49:44.991 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-03-07T02:49:45.017 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../025-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-03-07T02:49:45.049 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-03-07T02:49:45.152 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-singledispatch. 2024-03-07T02:49:45.179 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../039-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-03-07T02:49:45.218 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-03-07T02:49:45.344 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-cheroot. 2024-03-07T02:49:45.371 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../026-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-03-07T02:49:45.411 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-03-07T02:49:45.488 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-webob. 2024-03-07T02:49:45.503 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../040-python3-webob_1%3a1.8.6-1.1_all.deb ... 2024-03-07T02:49:45.538 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-webob (1:1.8.6-1.1) ... 2024-03-07T02:49:45.739 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-03-07T02:49:45.765 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../027-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-03-07T02:49:45.805 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-03-07T02:49:45.816 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-waitress. 2024-03-07T02:49:45.831 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../041-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-03-07T02:49:45.868 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-03-07T02:49:46.100 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-jaraco.text. 2024-03-07T02:49:46.126 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../028-python3-jaraco.text_3.6.0-2_all.deb ... 2024-03-07T02:49:46.166 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-03-07T02:49:46.177 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-tempita. 2024-03-07T02:49:46.203 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../042-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-03-07T02:49:46.244 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-03-07T02:49:46.428 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-03-07T02:49:46.454 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../029-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-03-07T02:49:46.486 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-03-07T02:49:46.521 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-paste. 2024-03-07T02:49:46.537 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../043-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-03-07T02:49:46.571 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-03-07T02:49:46.883 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-03-07T02:49:46.910 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../044-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-03-07T02:49:46.950 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-03-07T02:49:46.963 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-tempora. 2024-03-07T02:49:46.990 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../030-python3-tempora_4.1.2-1_all.deb ... 2024-03-07T02:49:47.030 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-03-07T02:49:47.219 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-pastedeploy. 2024-03-07T02:49:47.246 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../045-python3-pastedeploy_2.1.1-1_all.deb ... 2024-03-07T02:49:47.286 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-03-07T02:49:47.333 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-portend. 2024-03-07T02:49:47.360 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../031-python3-portend_3.0.0-1_all.deb ... 2024-03-07T02:49:47.399 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-03-07T02:49:47.572 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-webtest. 2024-03-07T02:49:47.599 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../046-python3-webtest_2.0.35-1_all.deb ... 2024-03-07T02:49:47.638 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-03-07T02:49:47.686 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-03-07T02:49:47.712 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../032-python3-zc.lockfile_2.0-1_all.deb ... 2024-03-07T02:49:47.752 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-03-07T02:49:47.950 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-pecan. 2024-03-07T02:49:47.976 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../047-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-03-07T02:49:48.005 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-cherrypy3. 2024-03-07T02:49:48.016 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-03-07T02:49:48.020 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../033-python3-cherrypy3_18.6.1-4_all.deb ... 2024-03-07T02:49:48.071 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-03-07T02:49:48.380 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-certifi. 2024-03-07T02:49:48.406 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../048-python3-certifi_2020.6.20-1_all.deb ... 2024-03-07T02:49:48.417 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-natsort. 2024-03-07T02:49:48.443 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../034-python3-natsort_8.0.2-1_all.deb ... 2024-03-07T02:49:48.446 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-03-07T02:49:48.483 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-03-07T02:49:48.766 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-idna. 2024-03-07T02:49:48.792 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../049-python3-idna_3.3-1_all.deb ... 2024-03-07T02:49:48.795 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-logutils. 2024-03-07T02:49:48.821 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../035-python3-logutils_0.3.3-8_all.deb ... 2024-03-07T02:49:48.824 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-idna (3.3-1) ... 2024-03-07T02:49:48.861 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-03-07T02:49:49.127 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-urllib3. 2024-03-07T02:49:49.153 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../050-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-03-07T02:49:49.164 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-markupsafe. 2024-03-07T02:49:49.191 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../036-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-03-07T02:49:49.193 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-03-07T02:49:49.231 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-03-07T02:49:49.484 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-mako. 2024-03-07T02:49:49.510 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../037-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-03-07T02:49:49.514 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-requests. 2024-03-07T02:49:49.540 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../051-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-03-07T02:49:49.543 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-03-07T02:49:49.580 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-03-07T02:49:49.837 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-simplegeneric. 2024-03-07T02:49:49.863 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../038-python3-simplegeneric_0.8.1-3_all.deb ... 2024-03-07T02:49:49.891 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-werkzeug. 2024-03-07T02:49:49.903 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-03-07T02:49:49.918 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../052-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.1_all.deb ... 2024-03-07T02:49:49.958 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ... 2024-03-07T02:49:50.190 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-singledispatch. 2024-03-07T02:49:50.216 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../039-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-03-07T02:49:50.236 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-03-07T02:49:50.252 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../053-ceph-mgr-modules-core_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:49:50.256 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-03-07T02:49:50.294 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-mgr-modules-core (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:50.666 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-03-07T02:49:50.692 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../054-libsqlite3-mod-ceph_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:50.724 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking libsqlite3-mod-ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:51.018 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-openssl. 2024-03-07T02:49:51.045 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../055-python3-openssl_21.0.0-1_all.deb ... 2024-03-07T02:49:51.085 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-03-07T02:49:51.260 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-webob. 2024-03-07T02:49:51.286 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../040-python3-webob_1%3a1.8.6-1.1_all.deb ... 2024-03-07T02:49:51.343 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-webob (1:1.8.6-1.1) ... 2024-03-07T02:49:51.662 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-waitress. 2024-03-07T02:49:51.689 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../041-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-03-07T02:49:51.724 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-03-07T02:49:51.755 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-mgr. 2024-03-07T02:49:51.782 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../056-ceph-mgr_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:51.990 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-tempita. 2024-03-07T02:49:52.016 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../042-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-03-07T02:49:52.048 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-03-07T02:49:52.181 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-mgr (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:52.335 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-paste. 2024-03-07T02:49:52.361 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../043-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-03-07T02:49:52.401 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-03-07T02:49:52.692 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-mon. 2024-03-07T02:49:52.696 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-03-07T02:49:52.707 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../057-ceph-mon_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:52.711 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../044-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-03-07T02:49:52.741 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-mon (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:52.745 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-03-07T02:49:52.990 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-pastedeploy. 2024-03-07T02:49:53.006 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../045-python3-pastedeploy_2.1.1-1_all.deb ... 2024-03-07T02:49:53.048 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-03-07T02:49:53.352 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-webtest. 2024-03-07T02:49:53.378 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../046-python3-webtest_2.0.35-1_all.deb ... 2024-03-07T02:49:53.418 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-03-07T02:49:53.621 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-osd. 2024-03-07T02:49:53.647 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../058-ceph-osd_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:53.686 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-osd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:53.729 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-pecan. 2024-03-07T02:49:53.756 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../047-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-03-07T02:49:53.796 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-03-07T02:49:54.151 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-certifi. 2024-03-07T02:49:54.177 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../048-python3-certifi_2020.6.20-1_all.deb ... 2024-03-07T02:49:54.217 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-03-07T02:49:54.504 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-idna. 2024-03-07T02:49:54.530 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../049-python3-idna_3.3-1_all.deb ... 2024-03-07T02:49:54.561 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-idna (3.3-1) ... 2024-03-07T02:49:54.831 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-urllib3. 2024-03-07T02:49:54.858 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../050-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-03-07T02:49:54.897 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-03-07T02:49:55.226 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-requests. 2024-03-07T02:49:55.253 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../051-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-03-07T02:49:55.301 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-03-07T02:49:55.515 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph. 2024-03-07T02:49:55.542 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../059-ceph_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:55.581 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:55.595 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-werkzeug. 2024-03-07T02:49:55.622 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../052-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.1_all.deb ... 2024-03-07T02:49:55.662 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ... 2024-03-07T02:49:55.892 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-fuse. 2024-03-07T02:49:55.919 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../060-ceph-fuse_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:55.950 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:55.965 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-03-07T02:49:55.980 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../053-ceph-mgr-modules-core_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:49:56.023 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-mgr-modules-core (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:56.313 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-mds. 2024-03-07T02:49:56.340 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../061-ceph-mds_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:56.378 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-03-07T02:49:56.379 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-mds (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:56.405 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../054-libsqlite3-mod-ceph_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:56.444 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking libsqlite3-mod-ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:56.714 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-openssl. 2024-03-07T02:49:56.741 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../055-python3-openssl_21.0.0-1_all.deb ... 2024-03-07T02:49:56.765 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package cephadm. 2024-03-07T02:49:56.792 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../062-cephadm_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:56.814 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-03-07T02:49:56.832 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:57.118 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-asyncssh. 2024-03-07T02:49:57.145 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../063-python3-asyncssh_2.5.0-1_all.deb ... 2024-03-07T02:49:57.184 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-asyncssh (2.5.0-1) ... 2024-03-07T02:49:57.308 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-mgr. 2024-03-07T02:49:57.336 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../056-ceph-mgr_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:57.375 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-mgr (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:57.513 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python-babel-localedata. 2024-03-07T02:49:57.528 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-03-07T02:49:57.571 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-03-07T02:49:57.770 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-mon. 2024-03-07T02:49:57.797 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../057-ceph-mon_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:57.837 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-mon (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:58.493 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-babel. 2024-03-07T02:49:58.520 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-03-07T02:49:58.560 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-03-07T02:49:58.790 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-osd. 2024-03-07T02:49:58.816 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../058-ceph-osd_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:49:58.856 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-osd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:58.879 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-jinja2. 2024-03-07T02:49:58.906 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.1_all.deb ... 2024-03-07T02:49:58.946 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.1) ... 2024-03-07T02:49:59.307 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-03-07T02:49:59.334 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:49:59.382 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-mgr-cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:49:59.727 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-bcrypt. 2024-03-07T02:49:59.754 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../068-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-03-07T02:49:59.794 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-03-07T02:50:00.055 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-pluggy. 2024-03-07T02:50:00.071 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../069-python3-pluggy_0.13.0-7.1_all.deb ... 2024-03-07T02:50:00.105 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-03-07T02:50:00.400 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-py. 2024-03-07T02:50:00.427 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../070-python3-py_1.10.0-1_all.deb ... 2024-03-07T02:50:00.466 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-py (1.10.0-1) ... 2024-03-07T02:50:00.762 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-iniconfig. 2024-03-07T02:50:00.789 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../071-python3-iniconfig_1.1.1-2_all.deb ... 2024-03-07T02:50:00.818 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph. 2024-03-07T02:50:00.828 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-03-07T02:50:00.833 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../059-ceph_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:00.868 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:01.073 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-toml. 2024-03-07T02:50:01.100 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../072-python3-toml_0.10.2-1_all.deb ... 2024-03-07T02:50:01.139 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-03-07T02:50:01.195 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-fuse. 2024-03-07T02:50:01.222 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../060-ceph-fuse_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:01.262 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:01.442 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-pytest. 2024-03-07T02:50:01.469 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../073-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-03-07T02:50:01.508 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-03-07T02:50:01.632 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-mds. 2024-03-07T02:50:01.658 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../061-ceph-mds_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:01.698 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-mds (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:01.829 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-repoze.lru. 2024-03-07T02:50:01.856 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../074-python3-repoze.lru_0.7-2_all.deb ... 2024-03-07T02:50:01.895 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-03-07T02:50:02.102 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package cephadm. 2024-03-07T02:50:02.128 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../062-cephadm_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:02.168 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:02.198 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-routes. 2024-03-07T02:50:02.225 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../075-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-03-07T02:50:02.265 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-03-07T02:50:02.430 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-asyncssh. 2024-03-07T02:50:02.456 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../063-python3-asyncssh_2.5.0-1_all.deb ... 2024-03-07T02:50:02.488 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-asyncssh (2.5.0-1) ... 2024-03-07T02:50:02.816 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python-babel-localedata. 2024-03-07T02:50:02.827 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-03-07T02:50:02.842 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-03-07T02:50:02.854 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../076-ceph-mgr-dashboard_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:50:02.874 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-03-07T02:50:02.893 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-mgr-dashboard (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:03.288 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-03-07T02:50:03.304 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../077-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-03-07T02:50:03.338 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-03-07T02:50:03.825 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-joblib. 2024-03-07T02:50:03.852 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../078-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-03-07T02:50:03.854 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-babel. 2024-03-07T02:50:03.867 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-03-07T02:50:03.892 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-03-07T02:50:03.903 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-03-07T02:50:04.199 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-jinja2. 2024-03-07T02:50:04.226 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.1_all.deb ... 2024-03-07T02:50:04.229 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-03-07T02:50:04.244 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../079-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-03-07T02:50:04.256 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.1) ... 2024-03-07T02:50:04.279 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-03-07T02:50:04.548 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-sklearn. 2024-03-07T02:50:04.575 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../080-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-03-07T02:50:04.610 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-03-07T02:50:04.615 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-03-07T02:50:04.637 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:50:04.677 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-mgr-cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:05.014 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-bcrypt. 2024-03-07T02:50:05.040 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../068-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-03-07T02:50:05.080 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-03-07T02:50:05.299 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-03-07T02:50:05.327 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../081-ceph-mgr-diskprediction-local_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:50:05.358 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-pluggy. 2024-03-07T02:50:05.365 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-mgr-diskprediction-local (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:05.384 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../069-python3-pluggy_0.13.0-7.1_all.deb ... 2024-03-07T02:50:05.424 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-03-07T02:50:05.720 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-py. 2024-03-07T02:50:05.746 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../070-python3-py_1.10.0-1_all.deb ... 2024-03-07T02:50:05.786 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-py (1.10.0-1) ... 2024-03-07T02:50:06.089 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-iniconfig. 2024-03-07T02:50:06.116 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../071-python3-iniconfig_1.1.1-2_all.deb ... 2024-03-07T02:50:06.156 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-03-07T02:50:06.301 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-cachetools. 2024-03-07T02:50:06.329 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../082-python3-cachetools_5.0.0-1_all.deb ... 2024-03-07T02:50:06.368 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-03-07T02:50:06.458 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-toml. 2024-03-07T02:50:06.485 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../072-python3-toml_0.10.2-1_all.deb ... 2024-03-07T02:50:06.550 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-03-07T02:50:06.781 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-pyasn1. 2024-03-07T02:50:06.809 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../083-python3-pyasn1_0.4.8-1_all.deb ... 2024-03-07T02:50:06.847 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-03-07T02:50:06.862 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-pytest. 2024-03-07T02:50:06.889 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../073-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-03-07T02:50:06.928 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-03-07T02:50:07.160 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-03-07T02:50:07.187 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../084-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-03-07T02:50:07.226 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-03-07T02:50:07.273 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-repoze.lru. 2024-03-07T02:50:07.289 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../074-python3-repoze.lru_0.7-2_all.deb ... 2024-03-07T02:50:07.331 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-03-07T02:50:07.521 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-rsa. 2024-03-07T02:50:07.549 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../085-python3-rsa_4.8-1_all.deb ... 2024-03-07T02:50:07.579 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-rsa (4.8-1) ... 2024-03-07T02:50:07.643 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-routes. 2024-03-07T02:50:07.670 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../075-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-03-07T02:50:07.709 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-03-07T02:50:08.055 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-03-07T02:50:08.082 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../076-ceph-mgr-dashboard_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:50:08.083 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-google-auth. 2024-03-07T02:50:08.110 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../086-python3-google-auth_1.5.1-3_all.deb ... 2024-03-07T02:50:08.121 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-mgr-dashboard (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:08.149 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-03-07T02:50:08.469 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-03-07T02:50:08.496 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../087-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-03-07T02:50:08.524 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-03-07T02:50:08.536 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-03-07T02:50:08.540 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../077-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-03-07T02:50:08.582 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-03-07T02:50:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-websocket. 2024-03-07T02:50:08.825 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../088-python3-websocket_1.2.3-1_all.deb ... 2024-03-07T02:50:08.855 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-03-07T02:50:09.103 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-joblib. 2024-03-07T02:50:09.130 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../078-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-03-07T02:50:09.142 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-kubernetes. 2024-03-07T02:50:09.169 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-03-07T02:50:09.170 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../089-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-03-07T02:50:09.209 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-03-07T02:50:09.481 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-03-07T02:50:09.509 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../079-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-03-07T02:50:09.539 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-03-07T02:50:09.814 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-03-07T02:50:09.831 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../090-ceph-mgr-k8sevents_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:50:09.872 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-mgr-k8sevents (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:09.892 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-sklearn. 2024-03-07T02:50:09.920 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../080-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-03-07T02:50:09.959 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-03-07T02:50:10.184 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package libonig5:amd64. 2024-03-07T02:50:10.211 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../091-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-03-07T02:50:10.250 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-03-07T02:50:10.587 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package libjq1:amd64. 2024-03-07T02:50:10.592 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-03-07T02:50:10.614 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../092-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-03-07T02:50:10.620 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../081-ceph-mgr-diskprediction-local_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:50:10.651 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-mgr-diskprediction-local (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:10.653 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-03-07T02:50:10.906 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package jq. 2024-03-07T02:50:10.934 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../093-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-03-07T02:50:10.972 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-03-07T02:50:11.200 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package socat. 2024-03-07T02:50:11.216 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../094-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-03-07T02:50:11.250 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-03-07T02:50:11.520 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package xmlstarlet. 2024-03-07T02:50:11.536 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../095-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-03-07T02:50:11.586 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-03-07T02:50:11.620 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-cachetools. 2024-03-07T02:50:11.647 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../082-python3-cachetools_5.0.0-1_all.deb ... 2024-03-07T02:50:11.687 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-03-07T02:50:12.032 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-test. 2024-03-07T02:50:12.057 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../096-ceph-test_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:12.090 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-test (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:12.200 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-pyasn1. 2024-03-07T02:50:12.227 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../083-python3-pyasn1_0.4.8-1_all.deb ... 2024-03-07T02:50:12.266 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-03-07T02:50:12.803 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-03-07T02:50:12.830 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../084-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-03-07T02:50:12.869 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-03-07T02:50:13.139 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-rsa. 2024-03-07T02:50:13.166 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../085-python3-rsa_4.8-1_all.deb ... 2024-03-07T02:50:13.197 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-rsa (4.8-1) ... 2024-03-07T02:50:13.493 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-google-auth. 2024-03-07T02:50:13.520 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../086-python3-google-auth_1.5.1-3_all.deb ... 2024-03-07T02:50:13.568 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-03-07T02:50:13.888 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-03-07T02:50:13.915 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../087-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-03-07T02:50:13.954 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-03-07T02:50:14.262 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package ceph-volume. 2024-03-07T02:50:14.274 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-websocket. 2024-03-07T02:50:14.289 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../097-ceph-volume_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:50:14.301 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../088-python3-websocket_1.2.3-1_all.deb ... 2024-03-07T02:50:14.328 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking ceph-volume (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:14.340 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-03-07T02:50:14.624 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package libcephfs-dev. 2024-03-07T02:50:14.640 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../098-libcephfs-dev_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:14.644 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-kubernetes. 2024-03-07T02:50:14.671 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../089-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-03-07T02:50:14.682 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking libcephfs-dev (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:14.702 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-03-07T02:50:14.994 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package nvme-cli. 2024-03-07T02:50:15.021 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../099-nvme-cli_1.16-3ubuntu0.1_amd64.deb ... 2024-03-07T02:50:15.052 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking nvme-cli (1.16-3ubuntu0.1) ... 2024-03-07T02:50:15.332 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-03-07T02:50:15.348 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../090-ceph-mgr-k8sevents_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:50:15.382 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-mgr-k8sevents (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:15.390 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python-asyncssh-doc. 2024-03-07T02:50:15.418 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../100-python-asyncssh-doc_2.5.0-1_all.deb ... 2024-03-07T02:50:15.456 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python-asyncssh-doc (2.5.0-1) ... 2024-03-07T02:50:15.710 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package libonig5:amd64. 2024-03-07T02:50:15.738 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../091-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-03-07T02:50:15.776 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-03-07T02:50:15.851 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-pastescript. 2024-03-07T02:50:15.879 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../101-python3-pastescript_2.0.2-4_all.deb ... 2024-03-07T02:50:15.918 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-03-07T02:50:16.096 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package libjq1:amd64. 2024-03-07T02:50:16.124 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../092-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-03-07T02:50:16.162 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-03-07T02:50:16.205 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-psutil. 2024-03-07T02:50:16.233 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../102-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-03-07T02:50:16.263 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-03-07T02:50:16.424 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package jq. 2024-03-07T02:50:16.451 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../093-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-03-07T02:50:16.490 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-03-07T02:50:16.591 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-pygments. 2024-03-07T02:50:16.619 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../103-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-03-07T02:50:16.657 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-03-07T02:50:16.701 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package socat. 2024-03-07T02:50:16.717 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../094-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-03-07T02:50:16.759 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-03-07T02:50:17.079 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package xmlstarlet. 2024-03-07T02:50:17.107 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../095-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-03-07T02:50:17.120 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-pyinotify. 2024-03-07T02:50:17.146 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-03-07T02:50:17.148 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../104-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-03-07T02:50:17.186 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-03-07T02:50:17.464 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package python3-simplejson. 2024-03-07T02:50:17.492 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../105-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-03-07T02:50:17.500 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-test. 2024-03-07T02:50:17.516 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../096-ceph-test_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:17.547 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-03-07T02:50:17.550 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-test (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:17.808 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-03-07T02:50:17.836 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../106-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-03-07T02:50:17.866 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-03-07T02:50:18.879 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package radosgw. 2024-03-07T02:50:18.907 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../107-radosgw_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:18.945 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking radosgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:19.665 INFO:teuthology.orchestra.run.smithi189.stdout:Selecting previously unselected package rbd-fuse. 2024-03-07T02:50:19.681 INFO:teuthology.orchestra.run.smithi189.stdout:Preparing to unpack .../108-rbd-fuse_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:19.723 INFO:teuthology.orchestra.run.smithi189.stdout:Unpacking rbd-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:19.779 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package ceph-volume. 2024-03-07T02:50:19.807 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../097-ceph-volume_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:50:19.845 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking ceph-volume (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:20.027 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-03-07T02:50:20.116 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package libcephfs-dev. 2024-03-07T02:50:20.144 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../098-libcephfs-dev_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:20.182 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking libcephfs-dev (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:20.277 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-03-07T02:50:20.406 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up nvme-cli (1.16-3ubuntu0.1) ... 2024-03-07T02:50:20.570 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package nvme-cli. 2024-03-07T02:50:20.597 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../099-nvme-cli_1.16-3ubuntu0.1_amd64.deb ... 2024-03-07T02:50:20.636 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking nvme-cli (1.16-3ubuntu0.1) ... 2024-03-07T02:50:20.647 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-03-07T02:50:20.960 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-03-07T02:50:20.974 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python-asyncssh-doc. 2024-03-07T02:50:20.999 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../100-python-asyncssh-doc_2.5.0-1_all.deb ... 2024-03-07T02:50:21.040 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python-asyncssh-doc (2.5.0-1) ... 2024-03-07T02:50:21.369 INFO:teuthology.orchestra.run.smithi189.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-03-07T02:50:21.443 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-pastescript. 2024-03-07T02:50:21.450 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:21.459 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../101-python3-pastescript_2.0.2-4_all.deb ... 2024-03-07T02:50:21.493 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-03-07T02:50:21.789 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-psutil. 2024-03-07T02:50:21.816 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../102-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-03-07T02:50:21.855 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-03-07T02:50:22.017 INFO:teuthology.orchestra.run.smithi189.stdout:Adding system user cephadm....done 2024-03-07T02:50:22.150 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-pygments. 2024-03-07T02:50:22.166 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../103-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-03-07T02:50:22.208 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-03-07T02:50:22.351 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-03-07T02:50:22.672 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-pyinotify. 2024-03-07T02:50:22.700 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../104-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-03-07T02:50:22.738 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-03-07T02:50:23.047 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-03-07T02:50:23.075 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package python3-simplejson. 2024-03-07T02:50:23.103 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../105-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-03-07T02:50:23.141 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-03-07T02:50:23.366 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python-asyncssh-doc (2.5.0-1) ... 2024-03-07T02:50:23.418 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-03-07T02:50:23.435 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../106-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-03-07T02:50:23.477 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-03-07T02:50:23.537 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-03-07T02:50:23.814 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-03-07T02:50:24.074 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-03-07T02:50:24.174 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-py (1.10.0-1) ... 2024-03-07T02:50:24.206 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package radosgw. 2024-03-07T02:50:24.234 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../107-radosgw_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:24.264 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking radosgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:24.459 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-03-07T02:50:24.819 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-03-07T02:50:24.992 INFO:teuthology.orchestra.run.smithi145.stdout:Selecting previously unselected package rbd-fuse. 2024-03-07T02:50:25.007 INFO:teuthology.orchestra.run.smithi145.stdout:Preparing to unpack .../108-rbd-fuse_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:25.050 INFO:teuthology.orchestra.run.smithi145.stdout:Unpacking rbd-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:25.079 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-03-07T02:50:25.355 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-03-07T02:50:25.362 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-03-07T02:50:25.613 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-03-07T02:50:25.614 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-03-07T02:50:25.731 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up nvme-cli (1.16-3ubuntu0.1) ... 2024-03-07T02:50:25.857 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-03-07T02:50:25.963 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-03-07T02:50:26.092 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-ceph-argparse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:26.290 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-03-07T02:50:26.326 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-03-07T02:50:26.435 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-03-07T02:50:26.686 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-03-07T02:50:26.719 INFO:teuthology.orchestra.run.smithi145.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-03-07T02:50:26.774 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:26.812 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-03-07T02:50:27.180 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-03-07T02:50:27.298 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-03-07T02:50:27.307 INFO:teuthology.orchestra.run.smithi145.stdout:Adding system user cephadm....done 2024-03-07T02:50:27.463 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-03-07T02:50:27.565 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-03-07T02:50:27.748 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-03-07T02:50:27.800 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-03-07T02:50:28.042 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python-asyncssh-doc (2.5.0-1) ... 2024-03-07T02:50:28.110 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-03-07T02:50:28.367 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-03-07T02:50:28.618 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-03-07T02:50:28.803 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-03-07T02:50:28.877 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-03-07T02:50:28.987 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-py (1.10.0-1) ... 2024-03-07T02:50:29.062 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-03-07T02:50:29.197 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-03-07T02:50:29.298 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-03-07T02:50:29.314 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-03-07T02:50:29.566 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-asyncssh (2.5.0-1) ... 2024-03-07T02:50:29.724 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-03-07T02:50:29.934 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-03-07T02:50:29.975 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-03-07T02:50:30.177 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-03-07T02:50:30.251 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-03-07T02:50:30.510 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-03-07T02:50:30.545 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-03-07T02:50:30.787 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-03-07T02:50:30.829 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ... 2024-03-07T02:50:31.030 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-ceph-argparse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:31.181 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-03-07T02:50:31.414 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-03-07T02:50:31.415 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-03-07T02:50:31.525 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:31.540 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-03-07T02:50:31.816 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-03-07T02:50:31.818 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-idna (3.3-1) ... 2024-03-07T02:50:31.925 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-03-07T02:50:32.161 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-03-07T02:50:32.285 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-03-07T02:50:32.410 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-03-07T02:50:32.679 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-03-07T02:50:32.930 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-03-07T02:50:33.214 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-03-07T02:50:33.341 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:50:33.476 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-toml (0.10.2-1) ... 2024-03-07T02:50:33.727 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-03-07T02:50:33.832 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-03-07T02:50:33.853 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-03-07T02:50:34.084 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-03-07T02:50:34.209 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-03-07T02:50:34.295 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-03-07T02:50:34.336 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-03-07T02:50:34.614 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-03-07T02:50:34.620 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-asyncssh (2.5.0-1) ... 2024-03-07T02:50:34.848 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-03-07T02:50:34.980 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-03-07T02:50:35.083 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:50:35.192 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-03-07T02:50:35.231 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-03-07T02:50:35.318 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-03-07T02:50:35.616 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-03-07T02:50:35.619 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-03-07T02:50:35.871 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-03-07T02:50:35.883 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ... 2024-03-07T02:50:36.130 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-03-07T02:50:36.251 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-03-07T02:50:36.389 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-03-07T02:50:36.520 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-03-07T02:50:36.615 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-03-07T02:50:36.620 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:36.859 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-03-07T02:50:36.905 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-idna (3.3-1) ... 2024-03-07T02:50:36.959 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-03-07T02:50:37.214 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-03-07T02:50:37.260 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-03-07T02:50:37.387 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-03-07T02:50:37.688 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-webob (1:1.8.6-1.1) ... 2024-03-07T02:50:37.997 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-03-07T02:50:38.232 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-03-07T02:50:38.395 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:50:38.530 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-toml (0.10.2-1) ... 2024-03-07T02:50:38.668 INFO:teuthology.orchestra.run.smithi189.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-03-07T02:50:38.780 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-03-07T02:50:38.901 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-03-07T02:50:38.906 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-03-07T02:50:39.032 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-03-07T02:50:39.052 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-03-07T02:50:39.317 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-03-07T02:50:39.454 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-03-07T02:50:39.585 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-03-07T02:50:39.722 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-portend (3.0.0-1) ... 2024-03-07T02:50:39.836 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:50:39.962 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-03-07T02:50:39.973 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:50:40.074 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.1) ... 2024-03-07T02:50:40.088 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-03-07T02:50:40.375 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-03-07T02:50:40.415 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-03-07T02:50:40.666 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-03-07T02:50:40.668 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-03-07T02:50:40.917 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-03-07T02:50:40.936 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-03-07T02:50:41.062 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-rsa (4.8-1) ... 2024-03-07T02:50:41.167 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-03-07T02:50:41.329 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-03-07T02:50:41.419 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-03-07T02:50:41.589 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-03-07T02:50:41.687 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-03-07T02:50:41.788 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-03-07T02:50:41.832 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-03-07T02:50:42.055 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-03-07T02:50:42.156 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-03-07T02:50:42.208 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-03-07T02:50:42.443 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-webob (1:1.8.6-1.1) ... 2024-03-07T02:50:42.501 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-03-07T02:50:42.742 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-03-07T02:50:42.862 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-03-07T02:50:42.988 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up librados2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:43.001 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-03-07T02:50:43.122 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up librgw2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:43.232 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up libsqlite3-mod-ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:43.333 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up libcephfs2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:43.391 INFO:teuthology.orchestra.run.smithi145.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-03-07T02:50:43.434 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up libradosstriper1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:43.637 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-03-07T02:50:43.793 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-03-07T02:50:43.913 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up librbd1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:43.972 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-03-07T02:50:44.048 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-mgr-modules-core (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:44.149 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:44.346 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-03-07T02:50:44.346 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-03-07T02:50:44.348 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-03-07T02:50:44.608 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-portend (3.0.0-1) ... 2024-03-07T02:50:44.800 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up libcephfs-dev (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:44.875 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:50:44.910 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-rados (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:45.001 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.1) ... 2024-03-07T02:50:45.036 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-03-07T02:50:45.328 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-03-07T02:50:45.612 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-03-07T02:50:45.905 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-03-07T02:50:46.031 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-rsa (4.8-1) ... 2024-03-07T02:50:46.240 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-rbd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:46.299 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-03-07T02:50:46.349 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up rbd-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:46.476 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-rgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:46.575 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-03-07T02:50:46.601 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up python3-cephfs (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:46.727 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:46.834 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-03-07T02:50:47.211 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-03-07T02:50:47.270 INFO:teuthology.orchestra.run.smithi189.stdout:Adding group ceph....done 2024-03-07T02:50:47.504 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-03-07T02:50:47.639 INFO:teuthology.orchestra.run.smithi189.stdout:Adding system user ceph....done 2024-03-07T02:50:47.807 INFO:teuthology.orchestra.run.smithi189.stdout:Setting system user ceph properties....done 2024-03-07T02:50:47.819 INFO:teuthology.orchestra.run.smithi189.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-03-07T02:50:47.856 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-03-07T02:50:47.953 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-03-07T02:50:47.957 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up librados2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:48.074 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up librgw2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:48.201 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up libsqlite3-mod-ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:48.255 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-03-07T02:50:48.326 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up libcephfs2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:48.461 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up libradosstriper1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:48.664 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-03-07T02:50:48.721 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-test (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:48.847 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up radosgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:48.932 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up librbd1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:49.092 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-mgr-modules-core (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:49.218 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:49.294 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-03-07T02:50:49.294 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-03-07T02:50:49.428 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-03-07T02:50:49.428 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-03-07T02:50:49.849 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-base (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:49.953 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up libcephfs-dev (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:50.088 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-rados (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:50.202 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-03-07T02:50:50.589 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-03-07T02:50:50.667 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-mds (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:50.903 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-03-07T02:50:50.903 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-03-07T02:50:51.386 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-mgr (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:51.622 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-03-07T02:50:51.623 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-03-07T02:50:51.793 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-rbd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:51.903 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up rbd-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:52.029 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-rgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:52.087 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-osd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:52.155 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up python3-cephfs (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:52.281 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:52.349 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-03-07T02:50:52.349 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-03-07T02:50:52.782 INFO:teuthology.orchestra.run.smithi145.stdout:Adding group ceph....done 2024-03-07T02:50:52.806 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-mgr-k8sevents (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:52.907 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-mgr-diskprediction-local (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:53.058 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-mon (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:53.176 INFO:teuthology.orchestra.run.smithi145.stdout:Adding system user ceph....done 2024-03-07T02:50:53.270 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-03-07T02:50:53.270 INFO:teuthology.orchestra.run.smithi189.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-03-07T02:50:53.319 INFO:teuthology.orchestra.run.smithi145.stdout:Setting system user ceph properties....done 2024-03-07T02:50:53.335 INFO:teuthology.orchestra.run.smithi145.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-03-07T02:50:53.473 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-03-07T02:50:53.717 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-mgr-cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:53.814 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-03-07T02:50:53.869 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:53.997 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-mgr-dashboard (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:54.146 INFO:teuthology.orchestra.run.smithi189.stdout:Setting up ceph-volume (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:54.307 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-test (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:54.433 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up radosgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:54.482 INFO:teuthology.orchestra.run.smithi189.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-03-07T02:50:54.897 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-03-07T02:50:54.897 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-03-07T02:50:55.327 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-base (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:55.358 INFO:teuthology.orchestra.run.smithi189.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-03-07T02:50:55.676 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-03-07T02:50:55.842 INFO:teuthology.orchestra.run.smithi189.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-03-07T02:50:55.847 DEBUG:teuthology.parallel:result is None 2024-03-07T02:50:56.128 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-mds (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:56.343 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-03-07T02:50:56.343 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-03-07T02:50:56.872 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-mgr (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:57.081 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-03-07T02:50:57.082 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-03-07T02:50:57.556 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-osd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:57.868 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-03-07T02:50:57.868 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-03-07T02:50:58.325 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-mgr-k8sevents (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:58.425 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-mgr-diskprediction-local (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:58.543 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-mon (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:58.755 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-03-07T02:50:58.755 INFO:teuthology.orchestra.run.smithi145.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-03-07T02:50:59.187 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-mgr-cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:59.596 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:59.722 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-mgr-dashboard (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:59.865 INFO:teuthology.orchestra.run.smithi145.stdout:Setting up ceph-volume (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:00.217 INFO:teuthology.orchestra.run.smithi145.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-03-07T02:51:01.293 INFO:teuthology.orchestra.run.smithi145.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-03-07T02:51:01.772 INFO:teuthology.orchestra.run.smithi145.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-03-07T02:51:01.776 DEBUG:teuthology.parallel:result is None 2024-03-07T02:51:01.777 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-07T02:51:01.777 INFO:teuthology.packaging:ref: None 2024-03-07T02:51:01.777 INFO:teuthology.packaging:tag: None 2024-03-07T02:51:01.777 INFO:teuthology.packaging:branch: reef 2024-03-07T02:51:01.777 INFO:teuthology.packaging:sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T02:51:01.777 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-03-07T02:51:02.025 DEBUG:teuthology.orchestra.run.smithi145:> dpkg-query -W -f '${Version}' ceph 2024-03-07T02:51:02.043 INFO:teuthology.orchestra.run.smithi145.stdout:18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:51:02.044 INFO:teuthology.packaging:The installed version of ceph is 18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:51:02.044 INFO:teuthology.task.install:The correct ceph version 18.2.1-707-gffcfe7db-1jammy is installed. 2024-03-07T02:51:02.045 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-07T02:51:02.045 INFO:teuthology.packaging:ref: None 2024-03-07T02:51:02.045 INFO:teuthology.packaging:tag: None 2024-03-07T02:51:02.046 INFO:teuthology.packaging:branch: reef 2024-03-07T02:51:02.046 INFO:teuthology.packaging:sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T02:51:02.046 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-03-07T02:51:02.408 DEBUG:teuthology.orchestra.run.smithi189:> dpkg-query -W -f '${Version}' ceph 2024-03-07T02:51:02.427 INFO:teuthology.orchestra.run.smithi189.stdout:18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:51:02.427 INFO:teuthology.packaging:The installed version of ceph is 18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:51:02.428 INFO:teuthology.task.install:The correct ceph version 18.2.1-707-gffcfe7db-1jammy is installed. 2024-03-07T02:51:02.429 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-03-07T02:51:02.429 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:51:02.429 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-07T02:51:02.444 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-07T02:51:02.444 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-07T02:51:02.484 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-03-07T02:51:02.484 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:51:02.485 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/usr/bin/daemon-helper 2024-03-07T02:51:02.500 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-07T02:51:02.556 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-07T02:51:02.556 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/usr/bin/daemon-helper 2024-03-07T02:51:02.568 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-07T02:51:02.623 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-03-07T02:51:02.623 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:51:02.623 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-07T02:51:02.636 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-07T02:51:02.691 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-07T02:51:02.691 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-07T02:51:02.705 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-07T02:51:02.759 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-03-07T02:51:02.759 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:51:02.759 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/usr/bin/stdin-killer 2024-03-07T02:51:02.772 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-07T02:51:02.827 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-07T02:51:02.827 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/usr/bin/stdin-killer 2024-03-07T02:51:02.840 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-07T02:51:02.895 INFO:teuthology.run_tasks:Running task cephadm... 2024-03-07T02:51:02.987 INFO:tasks.cephadm:Config: {'compiled_cephadm_branch': 'reef', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:reef', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'ea9a5688d9980e4ddedb59679d2960bcbac6fc71'} 2024-03-07T02:51:02.987 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:reef 2024-03-07T02:51:02.987 INFO:tasks.cephadm:Cluster fsid is 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:51:02.987 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-03-07T02:51:02.987 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.145', 'mon.c': '[v2:172.21.15.145:3301,v1:172.21.15.145:6790]', 'mon.b': '172.21.15.189'} 2024-03-07T02:51:02.988 INFO:tasks.cephadm:First mon is mon.a on smithi145 2024-03-07T02:51:02.988 INFO:tasks.cephadm:First mgr is y 2024-03-07T02:51:02.988 INFO:tasks.cephadm:Normalizing hostnames... 2024-03-07T02:51:02.988 DEBUG:teuthology.orchestra.run.smithi145:> sudo hostname $(hostname -s) 2024-03-07T02:51:03.002 DEBUG:teuthology.orchestra.run.smithi189:> sudo hostname $(hostname -s) 2024-03-07T02:51:03.017 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra for reef 2024-03-07T02:51:03.017 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T02:51:03.156 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': 'ea9a5688d9980e4ddedb59679d2960bcbac6fc71', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=jammy,DIST=jammy,MACHINE_SIZE=gigantic/54541/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.0.0-1905-gea9a5688', 'node_name': '172.21.2.17+braggi17', 'job_name': 'ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=jammy,DIST=jammy,MACHINE_SIZE=gigantic', 'package_manager_version': '19.0.0-1905-gea9a5688-1jammy'}, 'url': 'https://1.chacra.ceph.com/r/ceph/main/ea9a5688d9980e4ddedb59679d2960bcbac6fc71/ubuntu/jammy/flavors/default/', 'distro_codename': 'jammy', 'modified': '2024-03-06 23:55:50.104725', 'distro_version': '22.04', 'project': 'ceph', 'flavor': 'default', 'ref': 'main', 'chacra_url': 'https://1.chacra.ceph.com/repos/ceph/main/ea9a5688d9980e4ddedb59679d2960bcbac6fc71/ubuntu/jammy/flavors/default/', 'archs': ['x86_64'], 'distro': 'ubuntu'}] 2024-03-07T02:51:03.510 INFO:tasks.util.chacra:got chacra host 3.chacra.ceph.com, ref reef, sha1 ffcfe7db8c5ae0acba06074778b2c315e0dca035 from https://shaman.ceph.com/api/search/?project=ceph&distros=ubuntu%2F22.04%2Fx86_64&flavor=default&ref=reef 2024-03-07T02:51:03.512 INFO:tasks.cephadm:Discovered cachra url: https://3.chacra.ceph.com/binaries/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/x86_64/flavors/default/cephadm 2024-03-07T02:51:03.512 DEBUG:teuthology.orchestra.run.smithi145:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-07T02:51:04.377 INFO:teuthology.orchestra.run.smithi145.stdout:-rw-rw-r-- 1 ubuntu ubuntu 216271 Mar 7 02:51 /home/ubuntu/cephtest/cephadm 2024-03-07T02:51:04.377 DEBUG:teuthology.orchestra.run.smithi189:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-07T02:51:05.348 INFO:teuthology.orchestra.run.smithi189.stdout:-rw-rw-r-- 1 ubuntu ubuntu 216271 Mar 7 02:51 /home/ubuntu/cephtest/cephadm 2024-03-07T02:51:05.349 DEBUG:teuthology.orchestra.run.smithi145:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-07T02:51:05.357 DEBUG:teuthology.orchestra.run.smithi189:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-07T02:51:05.376 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:reef on all hosts... 2024-03-07T02:51:05.376 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-03-07T02:51:05.401 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-03-07T02:51:05.520 INFO:teuthology.orchestra.run.smithi145.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-03-07T02:51:05.526 INFO:teuthology.orchestra.run.smithi189.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-03-07T02:53:21.704 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-03-07T02:53:21.704 INFO:teuthology.orchestra.run.smithi189.stdout: "ceph_version": "ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable)", 2024-03-07T02:53:21.704 INFO:teuthology.orchestra.run.smithi189.stdout: "image_id": "0164bb8a7c6757b1a14e365a606a78ffe7cebb460f04a1ceb58f5dcf21071c24", 2024-03-07T02:53:21.704 INFO:teuthology.orchestra.run.smithi189.stdout: "repo_digests": [ 2024-03-07T02:53:21.705 INFO:teuthology.orchestra.run.smithi189.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4e788e3c6821efe986a710ffe10c052cd830c07b33cd4a66b97449fc9bbef8e7" 2024-03-07T02:53:21.705 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-03-07T02:53:21.705 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-03-07T02:55:29.429 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-03-07T02:55:29.430 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph_version": "ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable)", 2024-03-07T02:55:29.430 INFO:teuthology.orchestra.run.smithi145.stdout: "image_id": "0164bb8a7c6757b1a14e365a606a78ffe7cebb460f04a1ceb58f5dcf21071c24", 2024-03-07T02:55:29.430 INFO:teuthology.orchestra.run.smithi145.stdout: "repo_digests": [ 2024-03-07T02:55:29.430 INFO:teuthology.orchestra.run.smithi145.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4e788e3c6821efe986a710ffe10c052cd830c07b33cd4a66b97449fc9bbef8e7" 2024-03-07T02:55:29.430 INFO:teuthology.orchestra.run.smithi145.stdout: ] 2024-03-07T02:55:29.430 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-03-07T02:55:29.466 DEBUG:teuthology.orchestra.run.smithi145:> sudo mkdir -p /etc/ceph 2024-03-07T02:55:29.480 DEBUG:teuthology.orchestra.run.smithi189:> sudo mkdir -p /etc/ceph 2024-03-07T02:55:29.494 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod 777 /etc/ceph 2024-03-07T02:55:29.535 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod 777 /etc/ceph 2024-03-07T02:55:29.548 INFO:tasks.cephadm:Writing seed config... 2024-03-07T02:55:29.549 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-03-07T02:55:29.549 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-03-07T02:55:29.549 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-03-07T02:55:29.550 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-03-07T02:55:29.550 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-03-07T02:55:29.550 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-03-07T02:55:29.550 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-03-07T02:55:29.550 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-03-07T02:55:29.550 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-03-07T02:55:29.550 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-03-07T02:55:29.550 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-03-07T02:55:29.550 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:55:29.551 DEBUG:teuthology.orchestra.run.smithi145:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-03-07T02:55:29.583 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 8947a0f8-dc2d-11ee-95c6-87774f69a715 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-03-07T02:55:29.583 DEBUG:teuthology.orchestra.run.smithi145:mon.a> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.a.service 2024-03-07T02:55:29.628 DEBUG:teuthology.orchestra.run.smithi145:mgr.y> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.y.service 2024-03-07T02:55:29.672 INFO:tasks.cephadm:Bootstrapping... 2024-03-07T02:55:29.672 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef -v bootstrap --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.145 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-03-07T02:55:29.832 INFO:teuthology.orchestra.run.smithi145.stdout:-------------------------------------------------------------------------------- 2024-03-07T02:55:29.833 INFO:teuthology.orchestra.run.smithi145.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:reef', '-v', 'bootstrap', '--fsid', '8947a0f8-dc2d-11ee-95c6-87774f69a715', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.145', '--skip-admin-label'] 2024-03-07T02:55:29.833 INFO:teuthology.orchestra.run.smithi145.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-03-07T02:55:29.834 INFO:teuthology.orchestra.run.smithi145.stdout:Verifying podman|docker is present... 2024-03-07T02:55:29.834 INFO:teuthology.orchestra.run.smithi145.stdout:Verifying lvm2 is present... 2024-03-07T02:55:29.834 INFO:teuthology.orchestra.run.smithi145.stdout:Verifying time synchronization is in place... 2024-03-07T02:55:29.840 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-03-07T02:55:29.840 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-03-07T02:55:29.844 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-03-07T02:55:29.844 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout inactive 2024-03-07T02:55:29.847 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-03-07T02:55:29.848 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-03-07T02:55:29.851 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-03-07T02:55:29.851 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout inactive 2024-03-07T02:55:29.855 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-03-07T02:55:29.855 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout masked 2024-03-07T02:55:29.858 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-03-07T02:55:29.858 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout inactive 2024-03-07T02:55:29.861 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-03-07T02:55:29.861 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-03-07T02:55:29.864 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-03-07T02:55:29.864 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout inactive 2024-03-07T02:55:29.868 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout enabled 2024-03-07T02:55:29.872 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout active 2024-03-07T02:55:29.872 INFO:teuthology.orchestra.run.smithi145.stdout:Unit ntp.service is enabled and running 2024-03-07T02:55:29.872 INFO:teuthology.orchestra.run.smithi145.stdout:Repeating the final host check... 2024-03-07T02:55:29.872 INFO:teuthology.orchestra.run.smithi145.stdout:docker (/usr/bin/docker) is present 2024-03-07T02:55:29.872 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl is present 2024-03-07T02:55:29.872 INFO:teuthology.orchestra.run.smithi145.stdout:lvcreate is present 2024-03-07T02:55:29.875 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-03-07T02:55:29.875 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-03-07T02:55:29.878 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-03-07T02:55:29.878 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout inactive 2024-03-07T02:55:29.881 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-03-07T02:55:29.881 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-03-07T02:55:29.885 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-03-07T02:55:29.885 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout inactive 2024-03-07T02:55:29.888 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-03-07T02:55:29.888 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout masked 2024-03-07T02:55:29.891 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-03-07T02:55:29.891 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout inactive 2024-03-07T02:55:29.894 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-03-07T02:55:29.895 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-03-07T02:55:29.897 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-03-07T02:55:29.898 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout inactive 2024-03-07T02:55:29.901 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout enabled 2024-03-07T02:55:29.905 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stdout active 2024-03-07T02:55:29.905 INFO:teuthology.orchestra.run.smithi145.stdout:Unit ntp.service is enabled and running 2024-03-07T02:55:29.905 INFO:teuthology.orchestra.run.smithi145.stdout:Host looks OK 2024-03-07T02:55:29.905 INFO:teuthology.orchestra.run.smithi145.stdout:Cluster fsid: 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:55:29.905 INFO:teuthology.orchestra.run.smithi145.stdout:Acquiring lock 140210544330112 on /run/cephadm/8947a0f8-dc2d-11ee-95c6-87774f69a715.lock 2024-03-07T02:55:29.905 INFO:teuthology.orchestra.run.smithi145.stdout:Lock 140210544330112 acquired on /run/cephadm/8947a0f8-dc2d-11ee-95c6-87774f69a715.lock 2024-03-07T02:55:29.906 INFO:teuthology.orchestra.run.smithi145.stdout:Verifying IP 172.21.15.145 port 3300 ... 2024-03-07T02:55:29.906 INFO:teuthology.orchestra.run.smithi145.stdout:Verifying IP 172.21.15.145 port 6789 ... 2024-03-07T02:55:29.906 INFO:teuthology.orchestra.run.smithi145.stdout:Base mon IP(s) is [172.21.15.145:3300, 172.21.15.145:6789], mon addrv is [v2:172.21.15.145:3300,v1:172.21.15.145:6789] 2024-03-07T02:55:29.908 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 2024-03-07T02:55:29.909 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.145 metric 100 2024-03-07T02:55:29.909 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2024-03-07T02:55:29.909 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.145 metric 100 2024-03-07T02:55:29.909 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout 172.21.0.1 dev enp3s0f1 proto dhcp scope link src 172.21.15.145 metric 100 2024-03-07T02:55:29.909 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout 172.21.0.2 dev enp3s0f1 proto dhcp scope link src 172.21.15.145 metric 100 2024-03-07T02:55:29.909 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.145 metric 100 2024-03-07T02:55:29.910 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-03-07T02:55:29.910 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-03-07T02:55:29.910 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1782sec pref medium 2024-03-07T02:55:29.913 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-03-07T02:55:29.913 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout inet6 ::1/128 scope host 2024-03-07T02:55:29.913 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-03-07T02:55:29.914 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-03-07T02:55:29.914 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout inet6 fe80::ec4:7aff:fe88:7791/64 scope link 2024-03-07T02:55:29.914 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-03-07T02:55:29.914 INFO:teuthology.orchestra.run.smithi145.stdout:Mon IP `172.21.15.145` is in CIDR network `172.21.0.0/20` 2024-03-07T02:55:29.914 INFO:teuthology.orchestra.run.smithi145.stdout:Mon IP `172.21.15.145` is in CIDR network `172.21.0.0/20` 2024-03-07T02:55:29.914 INFO:teuthology.orchestra.run.smithi145.stdout:Mon IP `172.21.15.145` is in CIDR network `172.21.0.1/32` 2024-03-07T02:55:29.914 INFO:teuthology.orchestra.run.smithi145.stdout:Mon IP `172.21.15.145` is in CIDR network `172.21.0.1/32` 2024-03-07T02:55:29.914 INFO:teuthology.orchestra.run.smithi145.stdout:Mon IP `172.21.15.145` is in CIDR network `172.21.0.2/32` 2024-03-07T02:55:29.914 INFO:teuthology.orchestra.run.smithi145.stdout:Mon IP `172.21.15.145` is in CIDR network `172.21.0.2/32` 2024-03-07T02:55:29.914 INFO:teuthology.orchestra.run.smithi145.stdout:Mon IP `172.21.15.145` is in CIDR network `172.21.15.254/32` 2024-03-07T02:55:29.914 INFO:teuthology.orchestra.run.smithi145.stdout:Mon IP `172.21.15.145` is in CIDR network `172.21.15.254/32` 2024-03-07T02:55:29.914 INFO:teuthology.orchestra.run.smithi145.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.0.1/32', '172.21.0.1/32', '172.21.0.2/32', '172.21.0.2/32', '172.21.15.254/32', '172.21.15.254/32'] 2024-03-07T02:55:29.915 INFO:teuthology.orchestra.run.smithi145.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-03-07T02:55:29.915 INFO:teuthology.orchestra.run.smithi145.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-03-07T02:55:30.174 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/docker: stdout reef: Pulling from ceph-ci/ceph 2024-03-07T02:55:30.174 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/docker: stdout Digest: sha256:4e788e3c6821efe986a710ffe10c052cd830c07b33cd4a66b97449fc9bbef8e7 2024-03-07T02:55:30.174 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/docker: stdout Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:reef 2024-03-07T02:55:30.174 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/docker: stdout quay.ceph.io/ceph-ci/ceph:reef 2024-03-07T02:55:31.625 INFO:teuthology.orchestra.run.smithi145.stdout:ceph: stdout ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable) 2024-03-07T02:55:31.625 INFO:teuthology.orchestra.run.smithi145.stdout:Ceph version: ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable) 2024-03-07T02:55:31.625 INFO:teuthology.orchestra.run.smithi145.stdout:Extracting ceph user uid/gid from container image... 2024-03-07T02:55:32.971 INFO:teuthology.orchestra.run.smithi145.stdout:stat: stdout 167 167 2024-03-07T02:55:32.971 INFO:teuthology.orchestra.run.smithi145.stdout:Creating initial keys... 2024-03-07T02:55:34.151 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-authtool: stdout AQClLOlll09IJRAAN/E4XoEFkkRKkpv29cMVzw== 2024-03-07T02:55:35.397 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-authtool: stdout AQCmLOll9XFQNBAA9NMBrUS5UJP2zLpxA3Cz4Q== 2024-03-07T02:55:36.785 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-authtool: stdout AQCoLOll7CEdChAAkgTEShTAclFBEzvjqWMCGA== 2024-03-07T02:55:36.786 INFO:teuthology.orchestra.run.smithi145.stdout:Creating initial monmap... 2024-03-07T02:55:38.149 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-07T02:55:38.149 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2024-03-07T02:55:38.149 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:55:38.150 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-07T02:55:38.150 INFO:teuthology.orchestra.run.smithi145.stdout:monmaptool for a [v2:172.21.15.145:3300,v1:172.21.15.145:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-07T02:55:38.150 INFO:teuthology.orchestra.run.smithi145.stdout:setting min_mon_release = pacific 2024-03-07T02:55:38.150 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/monmaptool: set fsid to 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:55:38.150 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-07T02:55:38.150 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T02:55:38.150 INFO:teuthology.orchestra.run.smithi145.stdout:Creating mon... 2024-03-07T02:55:39.480 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.878+0000 7fbad372fc80 0 set uid:gid to 167:167 (ceph:ceph) 2024-03-07T02:55:39.480 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.878+0000 7fbad372fc80 1 imported monmap: 2024-03-07T02:55:39.480 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr epoch 0 2024-03-07T02:55:39.480 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:55:39.481 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr last_changed 2024-03-07T02:55:37.551545+0000 2024-03-07T02:55:39.481 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr created 2024-03-07T02:55:37.551545+0000 2024-03-07T02:55:39.481 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr min_mon_release 16 (pacific) 2024-03-07T02:55:39.481 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr election_strategy: 1 2024-03-07T02:55:39.481 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 0: [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon.a 2024-03-07T02:55:39.481 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.878+0000 7fbad372fc80 0 /usr/bin/ceph-mon: set fsid to 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:55:39.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: RocksDB version: 7.9.2 2024-03-07T02:55:39.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Git sha 0 2024-03-07T02:55:39.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Compile date 2024-03-06 23:19:49 2024-03-07T02:55:39.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: DB SUMMARY 2024-03-07T02:55:39.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: DB Session ID: 90IUZAQHO96ZZJGRDX22 2024-03-07T02:55:39.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2024-03-07T02:55:39.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2024-03-07T02:55:39.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.error_if_exists: 0 2024-03-07T02:55:39.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.create_if_missing: 1 2024-03-07T02:55:39.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.paranoid_checks: 1 2024-03-07T02:55:39.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-03-07T02:55:39.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-07T02:55:39.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-07T02:55:39.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.env: 0x5569d354c720 2024-03-07T02:55:39.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.fs: PosixFileSystem 2024-03-07T02:55:39.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.info_log: 0x5569d62b4720 2024-03-07T02:55:39.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_file_opening_threads: 16 2024-03-07T02:55:39.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.statistics: (nil) 2024-03-07T02:55:39.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.use_fsync: 0 2024-03-07T02:55:39.490 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_log_file_size: 0 2024-03-07T02:55:39.490 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-07T02:55:39.490 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-03-07T02:55:39.490 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.keep_log_file_num: 1000 2024-03-07T02:55:39.490 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.recycle_log_file_num: 0 2024-03-07T02:55:39.490 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.allow_fallocate: 1 2024-03-07T02:55:39.490 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.allow_mmap_reads: 0 2024-03-07T02:55:39.490 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.allow_mmap_writes: 0 2024-03-07T02:55:39.490 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.use_direct_reads: 0 2024-03-07T02:55:39.490 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-07T02:55:39.490 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.create_missing_column_families: 0 2024-03-07T02:55:39.490 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.db_log_dir: 2024-03-07T02:55:39.490 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.wal_dir: 2024-03-07T02:55:39.491 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-03-07T02:55:39.491 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-03-07T02:55:39.491 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-03-07T02:55:39.491 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-07T02:55:39.491 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-07T02:55:39.491 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-03-07T02:55:39.491 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.advise_random_on_open: 1 2024-03-07T02:55:39.491 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.db_write_buffer_size: 0 2024-03-07T02:55:39.491 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.write_buffer_manager: 0x5569d5545180 2024-03-07T02:55:39.491 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-07T02:55:39.491 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-07T02:55:39.491 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.rate_limiter: (nil) 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.wal_recovery_mode: 2 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.enable_thread_tracking: 0 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.enable_pipelined_write: 0 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.unordered_write: 0 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.row_cache: None 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.wal_filter: None 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-07T02:55:39.492 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.allow_ingest_behind: 0 2024-03-07T02:55:39.493 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.two_write_queues: 0 2024-03-07T02:55:39.493 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.manual_wal_flush: 0 2024-03-07T02:55:39.493 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.wal_compression: 0 2024-03-07T02:55:39.516 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.atomic_flush: 0 2024-03-07T02:55:39.516 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-07T02:55:39.516 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-03-07T02:55:39.516 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-03-07T02:55:39.516 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.log_readahead_size: 0 2024-03-07T02:55:39.516 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-07T02:55:39.516 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.best_efforts_recovery: 0 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.allow_data_in_errors: 0 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.db_host_id: __hostname__ 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_background_jobs: 2 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_background_compactions: -1 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_subcompactions: 1 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_total_wal_size: 0 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-03-07T02:55:39.517 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_open_files: -1 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.bytes_per_sync: 0 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compaction_readahead_size: 0 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_background_flushes: -1 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Compression algorithms supported: 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: kZSTD supported: 0 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: kXpressCompression supported: 0 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: kLZ4HCCompression supported: 1 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: kZlibCompression supported: 1 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: kSnappyCompression supported: 1 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: kLZ4Compression supported: 1 2024-03-07T02:55:39.518 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: kBZip2Compression supported: 0 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: [db/db_impl/db_impl_open.cc:317] Creating manifest 1 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.merge_operator: 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compaction_filter: None 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compaction_filter_factory: None 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.sst_partitioner_factory: None 2024-03-07T02:55:39.519 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5569d62b4840) 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks: 1 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks_with_high_priority: 0 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr pin_top_level_index_and_filter: 1 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr index_type: 0 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr data_block_index_type: 0 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr index_shortening: 1 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr data_block_hash_table_util_ratio: 0.750000 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr checksum: 4 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr no_block_cache: 0 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr block_cache: 0x5569d55c51f0 2024-03-07T02:55:39.520 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr block_cache_name: BinnedLRUCache 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr block_cache_options: 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr capacity : 536870912 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr num_shard_bits : 4 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr strict_capacity_limit : 0 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr high_pri_pool_ratio: 0.000 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr block_cache_compressed: (nil) 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr persistent_cache: (nil) 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr block_size: 4096 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr block_size_deviation: 10 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr block_restart_interval: 16 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr index_block_restart_interval: 1 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr metadata_block_size: 4096 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr partition_filters: 0 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr use_delta_encoding: 1 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr filter_policy: bloomfilter 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr whole_key_filtering: 1 2024-03-07T02:55:39.521 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr verify_compression: 0 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr read_amp_bytes_per_bit: 0 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr format_version: 5 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr enable_index_compression: 1 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr block_align: 0 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr max_auto_readahead_size: 262144 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr prepopulate_block_cache: 0 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr initial_auto_readahead_size: 8192 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr num_file_reads_for_auto_readahead: 2 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.write_buffer_size: 33554432 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_write_buffer_number: 2 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compression: NoCompression 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.bottommost_compression: Disabled 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.prefix_extractor: nullptr 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-07T02:55:39.522 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.num_levels: 7 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compression_opts.level: 32767 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compression_opts.strategy: 0 2024-03-07T02:55:39.523 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compression_opts.enabled: false 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.target_file_size_base: 67108864 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-07T02:55:39.524 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.arena_block_size: 1048576 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.disable_auto_compactions: 0 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-07T02:55:39.525 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.table_properties_collectors: 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.inplace_update_support: 0 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.bloom_locality: 0 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.max_successive_merges: 0 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.paranoid_file_checks: 0 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.force_consistency_checks: 1 2024-03-07T02:55:39.526 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.report_bg_io_stats: 0 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.ttl: 2592000 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.enable_blob_files: false 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.min_blob_size: 0 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.blob_file_size: 268435456 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.blob_file_starting_level: 0 2024-03-07T02:55:39.527 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: c046e38c-0770-4dfa-8688-b8a45c8198a5 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 5 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5569d563c000 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbad372fc80 4 rocksdb: DB pointer 0x5569d562e000 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbac392f700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.882+0000 7fbac392f700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr ** DB Stats ** 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T02:55:39.528 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-07T02:55:39.529 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Block cache BinnedLRUCache@0x5569d55c51f0#7 capacity: 512.00 MB usage: 0.00 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr Block cache entry stats(count,size,portion): Misc(1,0.00 KB,0%) 2024-03-07T02:55:39.530 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.531 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2024-03-07T02:55:39.531 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:55:39.531 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.886+0000 7fbad372fc80 4 rocksdb: [db/db_impl/db_impl.cc:496] Shutdown: canceling all background work 2024-03-07T02:55:39.531 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.886+0000 7fbad372fc80 4 rocksdb: [db/db_impl/db_impl.cc:704] Shutdown complete 2024-03-07T02:55:39.531 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:55:38.886+0000 7fbad372fc80 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2024-03-07T02:55:39.531 INFO:teuthology.orchestra.run.smithi145.stdout:create mon.a on 2024-03-07T02:55:39.798 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2024-03-07T02:55:39.999 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2024-03-07T02:55:40.243 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715.target -> /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715.target. 2024-03-07T02:55:40.243 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715.target -> /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715.target. 2024-03-07T02:55:40.525 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.a 2024-03-07T02:55:40.525 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Failed to reset failed state of unit ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.a.service: Unit ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.a.service not loaded. 2024-03-07T02:55:40.725 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715.target.wants/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.a.service -> /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service. 2024-03-07T02:55:40.734 INFO:teuthology.orchestra.run.smithi145.stdout:firewalld does not appear to be present 2024-03-07T02:55:40.734 INFO:teuthology.orchestra.run.smithi145.stdout:Not possible to enable service . firewalld.service is not available 2024-03-07T02:55:40.734 INFO:teuthology.orchestra.run.smithi145.stdout:Waiting for mon to start... 2024-03-07T02:55:40.734 INFO:teuthology.orchestra.run.smithi145.stdout:Waiting for mon... 2024-03-07T02:55:42.291 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:42 smithi145 bash[20854]: cluster 2024-03-07T02:55:42.014552+0000 mon.a (mon.0) 1 : cluster 1 mon.a is new leader, mons a in quorum (ranks 0) 2024-03-07T02:55:42.865 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout cluster: 2024-03-07T02:55:42.866 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout id: 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:55:42.866 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-03-07T02:55:42.866 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 2024-03-07T02:55:42.866 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout services: 2024-03-07T02:55:42.866 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.238734s) 2024-03-07T02:55:42.866 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-03-07T02:55:42.866 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-03-07T02:55:42.866 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 2024-03-07T02:55:42.866 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout data: 2024-03-07T02:55:42.867 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-03-07T02:55:42.867 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-03-07T02:55:42.867 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-03-07T02:55:42.867 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout pgs: 2024-03-07T02:55:42.868 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 2024-03-07T02:55:42.868 INFO:teuthology.orchestra.run.smithi145.stdout:mon is available 2024-03-07T02:55:42.868 INFO:teuthology.orchestra.run.smithi145.stdout:Assimilating anything we can from ceph.conf... 2024-03-07T02:55:43.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:43 smithi145 bash[20854]: cluster 2024-03-07T02:55:42.038494+0000 mon.a (mon.0) 2 : cluster 1 mon.a is new leader, mons a in quorum (ranks 0) 2024-03-07T02:55:43.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:43 smithi145 bash[20854]: cluster 2024-03-07T02:55:42.038716+0000 mon.a (mon.0) 3 : cluster 0 monmap e1: 1 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T02:55:43.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:43 smithi145 bash[20854]: cluster 2024-03-07T02:55:42.041148+0000 mon.a (mon.0) 4 : cluster 0 fsmap 2024-03-07T02:55:43.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:43 smithi145 bash[20854]: cluster 2024-03-07T02:55:42.043398+0000 mon.a (mon.0) 5 : cluster 0 osdmap e1: 0 total, 0 up, 0 in 2024-03-07T02:55:43.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:43 smithi145 bash[20854]: cluster 2024-03-07T02:55:42.043853+0000 mon.a (mon.0) 6 : cluster 0 mgrmap e1: no daemons active 2024-03-07T02:55:43.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:43 smithi145 bash[20854]: audit 2024-03-07T02:55:42.277176+0000 mon.a (mon.0) 7 : audit 0 from='client.? 172.21.15.145:0/1788304437' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-03-07T02:55:44.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:44 smithi145 bash[20854]: audit 2024-03-07T02:55:43.829683+0000 mon.a (mon.0) 8 : audit 1 from='client.? 172.21.15.145:0/1787067797' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-07T02:55:44.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:44 smithi145 bash[20854]: audit 2024-03-07T02:55:43.835260+0000 mon.a (mon.0) 9 : audit 1 from='client.? 172.21.15.145:0/1787067797' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-03-07T02:55:44.487 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 2024-03-07T02:55:44.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout [global] 2024-03-07T02:55:44.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout fsid = 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:55:44.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.145:3300,v1:172.21.15.145:6789] 2024-03-07T02:55:44.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-03-07T02:55:44.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-03-07T02:55:44.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-03-07T02:55:44.488 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-03-07T02:55:44.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 2024-03-07T02:55:44.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout [mgr] 2024-03-07T02:55:44.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-03-07T02:55:44.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 2024-03-07T02:55:44.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout [osd] 2024-03-07T02:55:44.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-03-07T02:55:44.489 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-03-07T02:55:44.490 INFO:teuthology.orchestra.run.smithi145.stdout:Generating new minimal ceph.conf... 2024-03-07T02:55:45.823 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:45 smithi145 bash[20854]: audit 2024-03-07T02:55:45.477187+0000 mon.a (mon.0) 10 : audit 0 from='client.? 172.21.15.145:0/1380479602' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:55:46.108 INFO:teuthology.orchestra.run.smithi145.stdout:Restarting the monitor... 2024-03-07T02:55:46.436 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:46 smithi145 systemd[1]: Stopping Ceph mon.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T02:55:46.436 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:46 smithi145 bash[20854]: debug 2024-03-07T02:55:46.238+0000 7f2a5547a700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T02:55:46.436 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:46 smithi145 bash[20854]: debug 2024-03-07T02:55:46.238+0000 7f2a5547a700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-03-07T02:55:47.035 INFO:teuthology.orchestra.run.smithi145.stdout:Setting public_network to 172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32 in mon config section 2024-03-07T02:55:47.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:46 smithi145 bash[21240]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mon-a 2024-03-07T02:55:47.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:47 smithi145 bash[21299]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mon-a 2024-03-07T02:55:47.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:47 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.a.service: Deactivated successfully. 2024-03-07T02:55:47.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:47 smithi145 systemd[1]: Stopped Ceph mon.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T02:55:47.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:47 smithi145 systemd[1]: Started Ceph mon.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T02:55:48.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.106+0000 7f2341839c80 0 set uid:gid to 167:167 (ceph:ceph) 2024-03-07T02:55:48.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.106+0000 7f2341839c80 0 ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable), process ceph-mon, pid 7 2024-03-07T02:55:48.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.106+0000 7f2341839c80 0 pidfile_write: ignore empty --pid-file 2024-03-07T02:55:48.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 0 load: jerasure load: lrc 2024-03-07T02:55:48.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: RocksDB version: 7.9.2 2024-03-07T02:55:48.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Git sha 0 2024-03-07T02:55:48.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Compile date 2024-03-06 23:19:49 2024-03-07T02:55:48.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: DB SUMMARY 2024-03-07T02:55:48.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: DB Session ID: 26OL9YCPH98XFU581YRS 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: CURRENT file: CURRENT 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: IDENTITY file: IDENTITY 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 84548 ; 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.error_if_exists: 0 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.create_if_missing: 0 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.paranoid_checks: 1 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.env: 0x55dc4629a720 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.fs: PosixFileSystem 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.info_log: 0x55dc48eb3500 2024-03-07T02:55:48.324 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_file_opening_threads: 16 2024-03-07T02:55:48.325 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.statistics: (nil) 2024-03-07T02:55:48.325 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.use_fsync: 0 2024-03-07T02:55:48.325 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_log_file_size: 0 2024-03-07T02:55:48.325 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.keep_log_file_num: 1000 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.recycle_log_file_num: 0 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.allow_fallocate: 1 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.allow_mmap_reads: 0 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.allow_mmap_writes: 0 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.use_direct_reads: 0 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.create_missing_column_families: 0 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.db_log_dir: 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.wal_dir: 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-03-07T02:55:48.326 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.advise_random_on_open: 1 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.db_write_buffer_size: 0 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.write_buffer_manager: 0x55dc48140320 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.rate_limiter: (nil) 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.wal_recovery_mode: 2 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.enable_thread_tracking: 0 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.enable_pipelined_write: 0 2024-03-07T02:55:48.327 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.unordered_write: 0 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.row_cache: None 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.wal_filter: None 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.allow_ingest_behind: 0 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.two_write_queues: 0 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.manual_wal_flush: 0 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.wal_compression: 0 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.atomic_flush: 0 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-03-07T02:55:48.328 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.log_readahead_size: 0 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.best_efforts_recovery: 0 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.allow_data_in_errors: 0 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.db_host_id: __hostname__ 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_background_jobs: 2 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_background_compactions: -1 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_subcompactions: 1 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-03-07T02:55:48.329 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_total_wal_size: 0 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_open_files: -1 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.bytes_per_sync: 0 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compaction_readahead_size: 0 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_background_flushes: -1 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Compression algorithms supported: 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: kZSTD supported: 0 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: kXpressCompression supported: 0 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: kLZ4HCCompression supported: 1 2024-03-07T02:55:48.330 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: kZlibCompression supported: 1 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: kSnappyCompression supported: 1 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: kLZ4Compression supported: 1 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: kBZip2Compression supported: 0 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.merge_operator: 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compaction_filter: None 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compaction_filter_factory: None 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.sst_partitioner_factory: None 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55dc48eb3600) 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: cache_index_and_filter_blocks: 1 2024-03-07T02:55:48.331 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: pin_top_level_index_and_filter: 1 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: index_type: 0 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: data_block_index_type: 0 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: index_shortening: 1 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: data_block_hash_table_util_ratio: 0.750000 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: checksum: 4 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: no_block_cache: 0 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: block_cache: 0x55dc481c31f0 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: block_cache_name: BinnedLRUCache 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: block_cache_options: 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: capacity : 536870912 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: num_shard_bits : 4 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: strict_capacity_limit : 0 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: high_pri_pool_ratio: 0.000 2024-03-07T02:55:48.332 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: block_cache_compressed: (nil) 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: persistent_cache: (nil) 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: block_size: 4096 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: block_size_deviation: 10 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: block_restart_interval: 16 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: index_block_restart_interval: 1 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: metadata_block_size: 4096 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: partition_filters: 0 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: use_delta_encoding: 1 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: filter_policy: bloomfilter 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: whole_key_filtering: 1 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: verify_compression: 0 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: read_amp_bytes_per_bit: 0 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: format_version: 5 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: enable_index_compression: 1 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: block_align: 0 2024-03-07T02:55:48.333 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: max_auto_readahead_size: 262144 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: prepopulate_block_cache: 0 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: initial_auto_readahead_size: 8192 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: num_file_reads_for_auto_readahead: 2 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.write_buffer_size: 33554432 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_write_buffer_number: 2 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compression: NoCompression 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.bottommost_compression: Disabled 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.prefix_extractor: nullptr 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.num_levels: 7 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-07T02:55:48.334 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compression_opts.level: 32767 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compression_opts.strategy: 0 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compression_opts.enabled: false 2024-03-07T02:55:48.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-07T02:55:48.336 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-07T02:55:48.336 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-07T02:55:48.336 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-07T02:55:48.336 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.target_file_size_base: 67108864 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.arena_block_size: 1048576 2024-03-07T02:55:48.337 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.disable_auto_compactions: 0 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.table_properties_collectors: 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.inplace_update_support: 0 2024-03-07T02:55:48.338 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.bloom_locality: 0 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.max_successive_merges: 0 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.paranoid_file_checks: 0 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.force_consistency_checks: 1 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.report_bg_io_stats: 0 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.ttl: 2592000 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.enable_blob_files: false 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.min_blob_size: 0 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.blob_file_size: 268435456 2024-03-07T02:55:48.339 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.blob_file_starting_level: 0 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: c046e38c-0770-4dfa-8688-b8a45c8198a5 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709780148117783, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.110+0000 7f2341839c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.114+0000 7f2341839c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709780148119396, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 81072, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 256, "table_properties": {"data_size": 79248, "index_size": 234, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73449, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 215, "num_filter_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1709780148, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "c046e38c-0770-4dfa-8688-b8a45c8198a5", "db_session_id": "26OL9YCPH98XFU581YRS", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.114+0000 7f2341839c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709780148119520, "job": 1, "event": "recovery_finished"} 2024-03-07T02:55:48.340 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.114+0000 7f2341839c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.114+0000 7f2341839c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.114+0000 7f2341839c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55dc48264000 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.114+0000 7f2341839c80 4 rocksdb: DB pointer 0x55dc48252000 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.114+0000 7f2330033700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.114+0000 7f2330033700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: ** DB Stats ** 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: ** Compaction Stats [default] ** 2024-03-07T02:55:48.341 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: L0 2/0 81.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 78.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Sum 2/0 81.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 78.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 78.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: ** Compaction Stats [default] ** 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 78.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Flush(GB): cumulative 0.000, interval 0.000 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: AddFile(Total Files): cumulative 0, interval 0 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: AddFile(L0 Files): cumulative 0, interval 0 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: AddFile(Keys): cumulative 0, interval 0 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Cumulative compaction: 0.00 GB write, 15.25 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T02:55:48.342 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Interval compaction: 0.00 GB write, 15.25 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Block cache BinnedLRUCache@0x55dc481c31f0#7 capacity: 512.00 MB usage: 1.19 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: Block cache entry stats(count,size,portion): FilterBlock(2,0.77 KB,0.000146031%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: ** File Read Latency Histogram By Level [default] ** 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.118+0000 7f2341839c80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] at bind addrs [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.118+0000 7f2341839c80 1 mon.a@-1(???) e1 preinit fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.118+0000 7f2341839c80 0 mon.a@-1(???).mds e1 new map 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.118+0000 7f2341839c80 0 mon.a@-1(???).mds e1 print_map 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: e1 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: enable_multiple, ever_enabled_multiple: 1,1 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: legacy client fscid: -1 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: No filesystems configured 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.118+0000 7f2341839c80 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-07T02:55:48.343 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.118+0000 7f2341839c80 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-07T02:55:48.344 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.118+0000 7f2341839c80 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-07T02:55:48.344 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.118+0000 7f2341839c80 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-07T02:55:48.344 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: debug 2024-03-07T02:55:48.118+0000 7f2341839c80 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-03-07T02:55:48.344 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: cluster 2024-03-07T02:55:48.146532+0000 mon.a (mon.0) 1 : cluster 1 mon.a is new leader, mons a in quorum (ranks 0) 2024-03-07T02:55:48.344 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: cluster 2024-03-07T02:55:48.146600+0000 mon.a (mon.0) 2 : cluster 0 monmap e1: 1 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T02:55:48.344 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: cluster 2024-03-07T02:55:48.147029+0000 mon.a (mon.0) 3 : cluster 0 fsmap 2024-03-07T02:55:48.344 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: cluster 2024-03-07T02:55:48.147055+0000 mon.a (mon.0) 4 : cluster 0 osdmap e1: 0 total, 0 up, 0 in 2024-03-07T02:55:48.344 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:48 smithi145 bash[21339]: cluster 2024-03-07T02:55:48.147456+0000 mon.a (mon.0) 5 : cluster 0 mgrmap e1: no daemons active 2024-03-07T02:55:49.118 INFO:teuthology.orchestra.run.smithi145.stdout:Wrote config to /etc/ceph/ceph.conf 2024-03-07T02:55:49.144 INFO:teuthology.orchestra.run.smithi145.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-03-07T02:55:49.144 INFO:teuthology.orchestra.run.smithi145.stdout:Creating mgr... 2024-03-07T02:55:49.144 INFO:teuthology.orchestra.run.smithi145.stdout:Verifying port 0.0.0.0:9283 ... 2024-03-07T02:55:49.144 INFO:teuthology.orchestra.run.smithi145.stdout:Verifying port 0.0.0.0:8765 ... 2024-03-07T02:55:49.478 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:55:49.478 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:49 smithi145 bash[21339]: audit 2024-03-07T02:55:48.388617+0000 mon.a (mon.0) 6 : audit 1 from='client.? 172.21.15.145:0/3543932184' entity='client.admin' 2024-03-07T02:55:49.478 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:55:49.486 INFO:teuthology.orchestra.run.smithi145.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.y 2024-03-07T02:55:49.486 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Failed to reset failed state of unit ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.y.service: Unit ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.y.service not loaded. 2024-03-07T02:55:49.702 INFO:teuthology.orchestra.run.smithi145.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715.target.wants/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.y.service -> /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service. 2024-03-07T02:55:49.711 INFO:teuthology.orchestra.run.smithi145.stdout:firewalld does not appear to be present 2024-03-07T02:55:49.711 INFO:teuthology.orchestra.run.smithi145.stdout:Not possible to enable service . firewalld.service is not available 2024-03-07T02:55:49.711 INFO:teuthology.orchestra.run.smithi145.stdout:firewalld does not appear to be present 2024-03-07T02:55:49.711 INFO:teuthology.orchestra.run.smithi145.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-03-07T02:55:49.711 INFO:teuthology.orchestra.run.smithi145.stdout:Waiting for mgr to start... 2024-03-07T02:55:49.711 INFO:teuthology.orchestra.run.smithi145.stdout:Waiting for mgr... 2024-03-07T02:55:49.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:55:49.750 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:55:49.750 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:55:49.750 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:55:49.750 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:49 smithi145 systemd[1]: Started Ceph mgr.y for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T02:55:51.268 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:51 smithi145 bash[21623]: debug 2024-03-07T02:55:51.118+0000 7f163bd9f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T02:55:51.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:51 smithi145 bash[21339]: audit 2024-03-07T02:55:51.253737+0000 mon.a (mon.0) 7 : audit 0 from='client.? 172.21.15.145:0/2361392989' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-07T02:55:51.893 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 2024-03-07T02:55:51.893 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:55:51.893 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "fsid": "8947a0f8-dc2d-11ee-95c6-87774f69a715", 2024-03-07T02:55:51.893 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "health": { 2024-03-07T02:55:51.893 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 0 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "a" 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "quorum_age": 3, 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-07T02:55:51.894 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-07T02:55:51.895 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-07T02:55:51.896 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-07T02:55:51.897 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "restful" 2024-03-07T02:55:51.897 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:55:51.897 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:55:51.897 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:51.897 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-07T02:55:51.897 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:55:51.897 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "modified": "2024-03-07T02:55:42.039965+0000", 2024-03-07T02:55:51.897 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:55:51.897 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:51.897 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-07T02:55:51.897 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:55:51.897 INFO:teuthology.orchestra.run.smithi145.stdout:mgr not available, waiting (1/15)... 2024-03-07T02:55:53.019 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:52 smithi145 bash[21623]: debug 2024-03-07T02:55:52.674+0000 7f163bd9f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T02:55:53.019 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:52 smithi145 bash[21623]: debug 2024-03-07T02:55:52.766+0000 7f163bd9f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T02:55:53.322 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:53 smithi145 bash[21623]: debug 2024-03-07T02:55:53.010+0000 7f163bd9f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T02:55:54.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:53 smithi145 bash[21623]: debug 2024-03-07T02:55:53.790+0000 7f163bd9f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T02:55:54.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:53 smithi145 bash[21623]: debug 2024-03-07T02:55:53.894+0000 7f163bd9f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T02:55:54.073 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:54 smithi145 bash[21623]: debug 2024-03-07T02:55:53.998+0000 7f163bd9f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T02:55:54.707 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:54 smithi145 bash[21623]: debug 2024-03-07T02:55:54.346+0000 7f163bd9f200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T02:55:54.707 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:54 smithi145 bash[21623]: debug 2024-03-07T02:55:54.458+0000 7f163bd9f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T02:55:54.707 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:54 smithi145 bash[21623]: debug 2024-03-07T02:55:54.562+0000 7f163bd9f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T02:55:55.047 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:54 smithi145 bash[21623]: debug 2024-03-07T02:55:54.914+0000 7f163bd9f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T02:55:55.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:55 smithi145 bash[21339]: audit 2024-03-07T02:55:55.032540+0000 mon.a (mon.0) 8 : audit 0 from='client.? 172.21.15.145:0/2233623099' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-07T02:55:55.323 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:55 smithi145 bash[21623]: debug 2024-03-07T02:55:55.158+0000 7f163bd9f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T02:55:55.593 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 2024-03-07T02:55:55.593 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:55:55.593 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "fsid": "8947a0f8-dc2d-11ee-95c6-87774f69a715", 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "health": { 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 0 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "a" 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-07T02:55:55.594 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:55:55.595 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-03-07T02:55:55.595 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-07T02:55:55.595 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:55.595 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-07T02:55:55.595 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:55:55.595 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-07T02:55:55.595 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-07T02:55:55.595 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-07T02:55:55.595 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-07T02:55:55.598 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-07T02:55:55.598 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-07T02:55:55.598 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:55.598 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-07T02:55:55.598 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-07T02:55:55.599 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "restful" 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "modified": "2024-03-07T02:55:42.039965+0000", 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:55:55.600 INFO:teuthology.orchestra.run.smithi145.stdout:mgr not available, waiting (2/15)... 2024-03-07T02:55:55.794 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:55 smithi145 bash[21623]: debug 2024-03-07T02:55:55.410+0000 7f163bd9f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T02:55:55.794 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:55 smithi145 bash[21623]: debug 2024-03-07T02:55:55.518+0000 7f163bd9f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T02:55:56.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:55 smithi145 bash[21623]: debug 2024-03-07T02:55:55.786+0000 7f163bd9f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T02:55:56.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:56 smithi145 bash[21623]: debug 2024-03-07T02:55:56.022+0000 7f163bd9f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:55:56.784 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:56 smithi145 bash[21623]: debug 2024-03-07T02:55:56.426+0000 7f163bd9f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T02:55:56.784 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:56 smithi145 bash[21623]: debug 2024-03-07T02:55:56.542+0000 7f163bd9f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T02:55:56.784 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:56 smithi145 bash[21623]: debug 2024-03-07T02:55:56.778+0000 7f163bd9f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:55:57.883 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:57 smithi145 bash[21623]: debug 2024-03-07T02:55:57.766+0000 7f163bd9f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T02:55:58.572 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:57 smithi145 bash[21623]: debug 2024-03-07T02:55:57.878+0000 7f163bd9f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T02:55:58.573 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:58 smithi145 bash[21623]: debug 2024-03-07T02:55:58.166+0000 7f163bd9f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T02:55:58.573 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:58 smithi145 bash[21623]: debug 2024-03-07T02:55:58.274+0000 7f163bd9f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T02:55:59.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:58 smithi145 bash[21623]: debug 2024-03-07T02:55:58.726+0000 7f163bd9f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T02:55:59.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:55:58 smithi145 bash[21623]: debug 2024-03-07T02:55:58.846+0000 7f163bd9f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T02:55:59.274 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 2024-03-07T02:55:59.274 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:55:59.274 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "fsid": "8947a0f8-dc2d-11ee-95c6-87774f69a715", 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "health": { 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 0 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "a" 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "quorum_age": 10, 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-07T02:55:59.275 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:55:59.276 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-03-07T02:55:59.276 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-07T02:55:59.276 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:59.276 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-07T02:55:59.276 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:55:59.276 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-07T02:55:59.276 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-07T02:55:59.276 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-07T02:55:59.276 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-07T02:55:59.279 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-07T02:55:59.279 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-07T02:55:59.279 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:59.280 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "restful" 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "modified": "2024-03-07T02:55:42.039965+0000", 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:55:59.281 INFO:teuthology.orchestra.run.smithi145.stdout:mgr not available, waiting (3/15)... 2024-03-07T02:55:59.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: audit 2024-03-07T02:55:58.663316+0000 mon.a (mon.0) 9 : audit 0 from='client.? 172.21.15.145:0/3330899501' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-07T02:55:59.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: cluster 2024-03-07T02:55:58.854448+0000 mon.a (mon.0) 10 : cluster 1 Activating manager daemon y 2024-03-07T02:55:59.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: cluster 2024-03-07T02:55:58.861061+0000 mon.a (mon.0) 11 : cluster 0 mgrmap e2: y(active, starting, since 0.00680998s) 2024-03-07T02:55:59.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: audit 2024-03-07T02:55:58.862066+0000 mon.a (mon.0) 12 : audit 0 from='mgr.14100 172.21.15.145:0/708436155' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T02:55:59.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: audit 2024-03-07T02:55:58.862185+0000 mon.a (mon.0) 13 : audit 0 from='mgr.14100 172.21.15.145:0/708436155' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T02:55:59.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: audit 2024-03-07T02:55:58.862278+0000 mon.a (mon.0) 14 : audit 0 from='mgr.14100 172.21.15.145:0/708436155' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T02:55:59.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: audit 2024-03-07T02:55:58.862377+0000 mon.a (mon.0) 15 : audit 0 from='mgr.14100 172.21.15.145:0/708436155' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T02:55:59.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: audit 2024-03-07T02:55:58.863256+0000 mon.a (mon.0) 16 : audit 0 from='mgr.14100 172.21.15.145:0/708436155' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T02:55:59.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: cluster 2024-03-07T02:55:58.887910+0000 mon.a (mon.0) 17 : cluster 1 Manager daemon y is now available 2024-03-07T02:55:59.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: audit 2024-03-07T02:55:58.899952+0000 mon.a (mon.0) 18 : audit 1 from='mgr.14100 172.21.15.145:0/708436155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T02:55:59.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: audit 2024-03-07T02:55:58.902804+0000 mon.a (mon.0) 19 : audit 1 from='mgr.14100 172.21.15.145:0/708436155' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T02:55:59.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: audit 2024-03-07T02:55:58.906703+0000 mon.a (mon.0) 20 : audit 1 from='mgr.14100 172.21.15.145:0/708436155' entity='mgr.y' 2024-03-07T02:55:59.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: audit 2024-03-07T02:55:58.908645+0000 mon.a (mon.0) 21 : audit 1 from='mgr.14100 172.21.15.145:0/708436155' entity='mgr.y' 2024-03-07T02:55:59.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:55:59 smithi145 bash[21339]: audit 2024-03-07T02:55:58.910580+0000 mon.a (mon.0) 22 : audit 1 from='mgr.14100 172.21.15.145:0/708436155' entity='mgr.y' 2024-03-07T02:56:01.301 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:00 smithi145 bash[21339]: cluster 2024-03-07T02:55:59.867662+0000 mon.a (mon.0) 23 : cluster 0 mgrmap e3: y(active, since 1.01342s) 2024-03-07T02:56:03.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:02 smithi145 bash[21339]: cluster 2024-03-07T02:56:01.869284+0000 mon.a (mon.0) 24 : cluster 0 mgrmap e4: y(active, since 3s) 2024-03-07T02:56:03.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:02 smithi145 bash[21339]: audit 2024-03-07T02:56:02.771937+0000 mon.a (mon.0) 25 : audit 0 from='client.? 172.21.15.145:0/1298317891' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-07T02:56:03.442 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 2024-03-07T02:56:03.442 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:56:03.442 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "fsid": "8947a0f8-dc2d-11ee-95c6-87774f69a715", 2024-03-07T02:56:03.442 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "health": { 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 0 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "a" 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "quorum_age": 14, 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-07T02:56:03.443 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-07T02:56:03.444 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "restful" 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:56:03.445 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:56:03.446 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:56:03.446 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-07T02:56:03.446 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:56:03.446 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "modified": "2024-03-07T02:55:42.039965+0000", 2024-03-07T02:56:03.446 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:56:03.446 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:56:03.446 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-07T02:56:03.446 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:56:03.446 INFO:teuthology.orchestra.run.smithi145.stdout:mgr is available 2024-03-07T02:56:05.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:04 smithi145 bash[21339]: audit 2024-03-07T02:56:04.684317+0000 mon.a (mon.0) 26 : audit 1 from='client.? 172.21.15.145:0/2614610899' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-07T02:56:05.310 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 2024-03-07T02:56:05.310 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout [global] 2024-03-07T02:56:05.310 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout fsid = 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:56:05.310 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-03-07T02:56:05.310 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-03-07T02:56:05.310 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-03-07T02:56:05.310 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-03-07T02:56:05.310 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 2024-03-07T02:56:05.310 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout [mgr] 2024-03-07T02:56:05.310 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-03-07T02:56:05.310 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 2024-03-07T02:56:05.311 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout [osd] 2024-03-07T02:56:05.311 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-03-07T02:56:05.311 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-03-07T02:56:05.311 INFO:teuthology.orchestra.run.smithi145.stdout:Enabling cephadm module... 2024-03-07T02:56:07.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:06 smithi145 bash[21339]: audit 2024-03-07T02:56:06.634364+0000 mon.a (mon.0) 27 : audit 1 from='client.? 172.21.15.145:0/149657287' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-03-07T02:56:07.073 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:06 smithi145 bash[21623]: ignoring --setuser ceph since I am not root 2024-03-07T02:56:07.073 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:06 smithi145 bash[21623]: ignoring --setgroup ceph since I am not root 2024-03-07T02:56:07.073 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:06 smithi145 bash[21623]: debug 2024-03-07T02:56:06.878+0000 7fa019b95200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T02:56:07.955 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:07 smithi145 bash[21339]: audit 2024-03-07T02:56:06.691528+0000 mon.a (mon.0) 28 : audit 1 from='client.? 172.21.15.145:0/149657287' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-03-07T02:56:07.955 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:07 smithi145 bash[21339]: cluster 2024-03-07T02:56:06.691624+0000 mon.a (mon.0) 29 : cluster 0 mgrmap e5: y(active, since 7s) 2024-03-07T02:56:08.800 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:08 smithi145 bash[21623]: debug 2024-03-07T02:56:08.446+0000 7fa019b95200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T02:56:08.800 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:08 smithi145 bash[21623]: debug 2024-03-07T02:56:08.542+0000 7fa019b95200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T02:56:09.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:08 smithi145 bash[21623]: debug 2024-03-07T02:56:08.794+0000 7fa019b95200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T02:56:09.261 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:56:09.261 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-03-07T02:56:09.261 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-07T02:56:09.261 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-03-07T02:56:09.261 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-03-07T02:56:09.261 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:56:09.261 INFO:teuthology.orchestra.run.smithi145.stdout:Waiting for the mgr to restart... 2024-03-07T02:56:09.261 INFO:teuthology.orchestra.run.smithi145.stdout:Waiting for mgr epoch 5... 2024-03-07T02:56:09.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:09 smithi145 bash[21339]: audit 2024-03-07T02:56:08.597494+0000 mon.a (mon.0) 30 : audit 0 from='client.? 172.21.15.145:0/2711990575' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-07T02:56:09.963 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:09 smithi145 bash[21623]: debug 2024-03-07T02:56:09.574+0000 7fa019b95200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T02:56:09.963 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:09 smithi145 bash[21623]: debug 2024-03-07T02:56:09.682+0000 7fa019b95200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T02:56:09.964 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:09 smithi145 bash[21623]: debug 2024-03-07T02:56:09.786+0000 7fa019b95200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T02:56:10.253 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:10 smithi145 bash[21623]: debug 2024-03-07T02:56:10.134+0000 7fa019b95200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T02:56:10.572 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:10 smithi145 bash[21623]: debug 2024-03-07T02:56:10.246+0000 7fa019b95200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T02:56:10.572 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:10 smithi145 bash[21623]: debug 2024-03-07T02:56:10.346+0000 7fa019b95200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T02:56:11.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:10 smithi145 bash[21623]: debug 2024-03-07T02:56:10.694+0000 7fa019b95200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T02:56:11.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:10 smithi145 bash[21623]: debug 2024-03-07T02:56:10.930+0000 7fa019b95200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T02:56:11.572 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:11 smithi145 bash[21623]: debug 2024-03-07T02:56:11.186+0000 7fa019b95200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T02:56:11.572 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:11 smithi145 bash[21623]: debug 2024-03-07T02:56:11.298+0000 7fa019b95200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T02:56:12.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:11 smithi145 bash[21623]: debug 2024-03-07T02:56:11.566+0000 7fa019b95200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T02:56:12.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:11 smithi145 bash[21623]: debug 2024-03-07T02:56:11.802+0000 7fa019b95200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:56:12.568 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:12 smithi145 bash[21623]: debug 2024-03-07T02:56:12.206+0000 7fa019b95200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T02:56:12.568 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:12 smithi145 bash[21623]: debug 2024-03-07T02:56:12.322+0000 7fa019b95200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T02:56:12.568 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:12 smithi145 bash[21623]: debug 2024-03-07T02:56:12.562+0000 7fa019b95200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:56:13.973 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:13 smithi145 bash[21623]: debug 2024-03-07T02:56:13.566+0000 7fa019b95200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T02:56:13.974 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:13 smithi145 bash[21623]: debug 2024-03-07T02:56:13.682+0000 7fa019b95200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T02:56:14.322 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:13 smithi145 bash[21623]: debug 2024-03-07T02:56:13.966+0000 7fa019b95200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T02:56:14.322 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:14 smithi145 bash[21623]: debug 2024-03-07T02:56:14.078+0000 7fa019b95200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T02:56:14.823 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:14 smithi145 bash[21623]: debug 2024-03-07T02:56:14.518+0000 7fa019b95200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T02:56:14.824 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:14 smithi145 bash[21623]: debug 2024-03-07T02:56:14.634+0000 7fa019b95200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T02:56:14.824 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:14 smithi145 bash[21339]: cluster 2024-03-07T02:56:14.659748+0000 mon.a (mon.0) 31 : cluster 1 Active manager daemon y restarted 2024-03-07T02:56:14.824 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:14 smithi145 bash[21339]: cluster 2024-03-07T02:56:14.660123+0000 mon.a (mon.0) 32 : cluster 1 Activating manager daemon y 2024-03-07T02:56:14.824 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:14 smithi145 bash[21339]: cluster 2024-03-07T02:56:14.666715+0000 mon.a (mon.0) 33 : cluster 0 osdmap e2: 0 total, 0 up, 0 in 2024-03-07T02:56:14.824 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:14 smithi145 bash[21339]: cluster 2024-03-07T02:56:14.666968+0000 mon.a (mon.0) 34 : cluster 0 mgrmap e6: y(active, starting, since 0.00694219s) 2024-03-07T02:56:14.824 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:14 smithi145 bash[21339]: audit 2024-03-07T02:56:14.669526+0000 mon.a (mon.0) 35 : audit 0 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T02:56:14.824 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:14 smithi145 bash[21339]: audit 2024-03-07T02:56:14.671460+0000 mon.a (mon.0) 36 : audit 0 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T02:56:14.824 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:14 smithi145 bash[21339]: audit 2024-03-07T02:56:14.672510+0000 mon.a (mon.0) 37 : audit 0 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T02:56:14.824 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:14 smithi145 bash[21339]: audit 2024-03-07T02:56:14.672733+0000 mon.a (mon.0) 38 : audit 0 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T02:56:14.824 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:14 smithi145 bash[21339]: audit 2024-03-07T02:56:14.672896+0000 mon.a (mon.0) 39 : audit 0 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T02:56:14.825 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:14 smithi145 bash[21339]: cluster 2024-03-07T02:56:14.700179+0000 mon.a (mon.0) 40 : cluster 1 Manager daemon y is now available 2024-03-07T02:56:16.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:15 smithi145 bash[21339]: cephadm 2024-03-07T02:56:14.714449+0000 mgr.y (mgr.14120) 1 : cephadm 1 Found migration_current of "None". Setting to last migration. 2024-03-07T02:56:16.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:15 smithi145 bash[21339]: audit 2024-03-07T02:56:14.716816+0000 mon.a (mon.0) 41 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:16.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:15 smithi145 bash[21339]: audit 2024-03-07T02:56:14.720288+0000 mon.a (mon.0) 42 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:16.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:15 smithi145 bash[21339]: audit 2024-03-07T02:56:14.724603+0000 mon.a (mon.0) 43 : audit 0 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:56:16.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:15 smithi145 bash[21339]: audit 2024-03-07T02:56:14.726432+0000 mon.a (mon.0) 44 : audit 0 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:56:16.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:15 smithi145 bash[21339]: audit 2024-03-07T02:56:14.730707+0000 mon.a (mon.0) 45 : audit 0 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:56:16.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:15 smithi145 bash[21339]: audit 2024-03-07T02:56:14.739432+0000 mon.a (mon.0) 46 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T02:56:16.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:15 smithi145 bash[21339]: audit 2024-03-07T02:56:14.741468+0000 mon.a (mon.0) 47 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T02:56:16.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:15 smithi145 bash[21339]: audit 2024-03-07T02:56:15.514142+0000 mon.a (mon.0) 48 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:16.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:15 smithi145 bash[21339]: audit 2024-03-07T02:56:15.517664+0000 mon.a (mon.0) 49 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:16.074 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:15 smithi145 bash[21339]: cluster 2024-03-07T02:56:15.671498+0000 mon.a (mon.0) 50 : cluster 0 mgrmap e7: y(active, since 1.01147s) 2024-03-07T02:56:16.383 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:56:16.383 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-03-07T02:56:16.383 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "initialized": true 2024-03-07T02:56:16.383 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:56:16.383 INFO:teuthology.orchestra.run.smithi145.stdout:mgr epoch 5 is available 2024-03-07T02:56:16.383 INFO:teuthology.orchestra.run.smithi145.stdout:Setting orchestrator backend to cephadm... 2024-03-07T02:56:17.055 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:16 smithi145 bash[21339]: audit 2024-03-07T02:56:15.669244+0000 mgr.y (mgr.14120) 2 : audit 0 from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-07T02:56:17.055 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:16 smithi145 bash[21339]: audit 2024-03-07T02:56:15.683807+0000 mgr.y (mgr.14120) 3 : audit 0 from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-07T02:56:17.055 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:16 smithi145 bash[21339]: audit 2024-03-07T02:56:16.251718+0000 mon.a (mon.0) 51 : audit 0 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:56:18.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:17 smithi145 bash[21339]: cephadm 2024-03-07T02:56:16.001097+0000 mgr.y (mgr.14120) 4 : cephadm 1 [07/Mar/2024:02:56:16] ENGINE Bus STARTING 2024-03-07T02:56:18.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:17 smithi145 bash[21339]: cephadm 2024-03-07T02:56:16.104081+0000 mgr.y (mgr.14120) 5 : cephadm 1 [07/Mar/2024:02:56:16] ENGINE Serving on http://172.21.15.145:8765 2024-03-07T02:56:18.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:17 smithi145 bash[21339]: cephadm 2024-03-07T02:56:16.221885+0000 mgr.y (mgr.14120) 6 : cephadm 1 [07/Mar/2024:02:56:16] ENGINE Serving on https://172.21.15.145:7150 2024-03-07T02:56:18.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:17 smithi145 bash[21339]: cephadm 2024-03-07T02:56:16.222137+0000 mgr.y (mgr.14120) 7 : cephadm 1 [07/Mar/2024:02:56:16] ENGINE Bus STARTED 2024-03-07T02:56:18.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:17 smithi145 bash[21339]: cluster 2024-03-07T02:56:16.727346+0000 mon.a (mon.0) 52 : cluster 0 mgrmap e8: y(active, since 2s) 2024-03-07T02:56:18.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:17 smithi145 bash[21339]: audit 2024-03-07T02:56:17.625435+0000 mon.a (mon.0) 53 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:18.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:17 smithi145 bash[21339]: audit 2024-03-07T02:56:17.672472+0000 mon.a (mon.0) 54 : audit 0 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:56:19.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:18 smithi145 bash[21339]: audit 2024-03-07T02:56:17.619545+0000 mgr.y (mgr.14120) 8 : audit 0 from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:56:20.037 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout value unchanged 2024-03-07T02:56:20.037 INFO:teuthology.orchestra.run.smithi145.stdout:Generating ssh key... 2024-03-07T02:56:20.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:19 smithi145 bash[21339]: audit 2024-03-07T02:56:19.447183+0000 mgr.y (mgr.14120) 9 : audit 0 from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:56:21.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: Generating public/private rsa key pair. 2024-03-07T02:56:21.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: Your identification has been saved in /tmp/tmpv4k2qmj1/key. 2024-03-07T02:56:21.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: Your public key has been saved in /tmp/tmpv4k2qmj1/key.pub. 2024-03-07T02:56:21.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: The key fingerprint is: 2024-03-07T02:56:21.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: SHA256:+TZn7dETDbz/lzyezvLJtINl6vFNui0ZBAYybeIxFWY ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:56:21.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: The key's randomart image is: 2024-03-07T02:56:21.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: +---[RSA 3072]----+ 2024-03-07T02:56:21.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: | ooEo | 2024-03-07T02:56:21.624 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: | +=o o. | 2024-03-07T02:56:21.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: | . = . .o | 2024-03-07T02:56:21.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: | .. .o.| 2024-03-07T02:56:21.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: | S ....| 2024-03-07T02:56:21.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: | . ..=.| 2024-03-07T02:56:21.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: | + o.O==| 2024-03-07T02:56:21.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: | . + =OXO| 2024-03-07T02:56:21.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: | ..O&B| 2024-03-07T02:56:21.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:21 smithi145 bash[21623]: +----[SHA256]-----+ 2024-03-07T02:56:22.896 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:22 smithi145 bash[21339]: audit 2024-03-07T02:56:21.290489+0000 mgr.y (mgr.14120) 10 : audit 0 from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:56:22.896 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:22 smithi145 bash[21339]: cephadm 2024-03-07T02:56:21.290909+0000 mgr.y (mgr.14120) 11 : cephadm 1 Generating ssh key... 2024-03-07T02:56:22.897 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:22 smithi145 bash[21339]: audit 2024-03-07T02:56:21.626989+0000 mon.a (mon.0) 55 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:22.897 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:22 smithi145 bash[21339]: audit 2024-03-07T02:56:21.629538+0000 mon.a (mon.0) 56 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:24.039 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDojO7HJbq4Kxb+/Tnl+CCWG5QEARyXEL/prNEEYqkUT1QBJASTMqWthwmets1IdPWFbxKzspwvrG7QSfSxV3b74SxAesK8V/6u+42PosIiBhfq2p1qpMKlGRgx2CaPKsXdGD4jQm1iWiTvyObi8TXQYyqu/jE8xp01mSgQk4cYhh/prWQ7vEZ1moEG0sQlRb86bAyt2U8w/5HLdI8T+Txek37nKrj7f0ff01AlIPRhaRQYzkebCxYTaYV80Le9LI/LYcyRsJYfThBZO6u0YR/5EC14kbqOkT9wCNL/VSB0eLLUDw+SipCWLWwvRTk2eG7ngnDxM7Z/y0i8UZdVVYwXdQMFdLXZl92lEP7o8nlFNqQqfdX+R/uVp22w8DrIy+jhwQZ2vwBoM3gNUhBSdI0veeqGXo+XZ+1n8UDBwpq99x/4mgwGBjDZmUnvqpa3k1z8CaulKzdp30R1FlX5wgd34RkyvUFWvuPXNo1n5ReXjW3PJjEn00SHRCLL0DZ9pR0= ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:56:24.039 INFO:teuthology.orchestra.run.smithi145.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-03-07T02:56:24.039 INFO:teuthology.orchestra.run.smithi145.stdout:Adding key to root@localhost authorized_keys... 2024-03-07T02:56:24.039 INFO:teuthology.orchestra.run.smithi145.stdout:Adding host smithi145... 2024-03-07T02:56:24.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:24 smithi145 bash[21339]: audit 2024-03-07T02:56:23.472323+0000 mgr.y (mgr.14120) 12 : audit 0 from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:56:25.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:25 smithi145 bash[21339]: audit 2024-03-07T02:56:25.284747+0000 mgr.y (mgr.14120) 13 : audit 0 from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi145", "addr": "172.21.15.145", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:56:27.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:26 smithi145 bash[21339]: cephadm 2024-03-07T02:56:25.816500+0000 mgr.y (mgr.14120) 14 : cephadm 1 Deploying cephadm binary to smithi145 2024-03-07T02:56:29.054 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout Added host 'smithi145' with addr '172.21.15.145' 2024-03-07T02:56:29.054 INFO:teuthology.orchestra.run.smithi145.stdout:Deploying unmanaged mon service... 2024-03-07T02:56:29.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:29 smithi145 bash[21339]: audit 2024-03-07T02:56:28.195677+0000 mon.a (mon.0) 57 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:29.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:29 smithi145 bash[21339]: cephadm 2024-03-07T02:56:28.196104+0000 mgr.y (mgr.14120) 15 : cephadm 1 Added host smithi145 2024-03-07T02:56:29.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:29 smithi145 bash[21339]: audit 2024-03-07T02:56:28.239633+0000 mon.a (mon.0) 58 : audit 0 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:56:31.383 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-03-07T02:56:31.383 INFO:teuthology.orchestra.run.smithi145.stdout:Deploying unmanaged mgr service... 2024-03-07T02:56:32.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:31 smithi145 bash[21339]: audit 2024-03-07T02:56:30.779842+0000 mgr.y (mgr.14120) 16 : audit 0 from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:56:32.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:31 smithi145 bash[21339]: cephadm 2024-03-07T02:56:30.782945+0000 mgr.y (mgr.14120) 17 : cephadm 1 Saving service mon spec with placement count:5 2024-03-07T02:56:32.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:31 smithi145 bash[21339]: audit 2024-03-07T02:56:30.787984+0000 mon.a (mon.0) 59 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:32.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:31 smithi145 bash[21339]: audit 2024-03-07T02:56:30.916014+0000 mon.a (mon.0) 60 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:32.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:31 smithi145 bash[21339]: audit 2024-03-07T02:56:31.243328+0000 mon.a (mon.0) 61 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:33.423 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-03-07T02:56:34.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:33 smithi145 bash[21339]: audit 2024-03-07T02:56:32.756697+0000 mgr.y (mgr.14120) 18 : audit 0 from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:56:34.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:33 smithi145 bash[21339]: cephadm 2024-03-07T02:56:32.759187+0000 mgr.y (mgr.14120) 19 : cephadm 1 Saving service mgr spec with placement count:2 2024-03-07T02:56:34.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:33 smithi145 bash[21339]: audit 2024-03-07T02:56:32.763833+0000 mon.a (mon.0) 62 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:36.027 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:35 smithi145 bash[21339]: cluster 2024-03-07T02:56:34.674114+0000 mgr.y (mgr.14120) 20 : cluster 0 pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:56:36.028 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:35 smithi145 bash[21339]: audit 2024-03-07T02:56:34.735409+0000 mon.a (mon.0) 63 : audit 1 from='client.? 172.21.15.145:0/617734775' entity='client.admin' 2024-03-07T02:56:37.222 INFO:teuthology.orchestra.run.smithi145.stdout:Enabling the dashboard module... 2024-03-07T02:56:37.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:37 smithi145 bash[21339]: audit 2024-03-07T02:56:36.273471+0000 mon.a (mon.0) 64 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:37.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:37 smithi145 bash[21339]: audit 2024-03-07T02:56:36.609192+0000 mon.a (mon.0) 65 : audit 1 from='client.? 172.21.15.145:0/1517775743' entity='client.admin' 2024-03-07T02:56:37.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:37 smithi145 bash[21339]: cluster 2024-03-07T02:56:36.674335+0000 mgr.y (mgr.14120) 21 : cluster 0 pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:56:37.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:37 smithi145 bash[21339]: audit 2024-03-07T02:56:36.967165+0000 mon.a (mon.0) 66 : audit 1 from='mgr.14120 172.21.15.145:0/1144209844' entity='mgr.y' 2024-03-07T02:56:39.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:39 smithi145 bash[21339]: audit 2024-03-07T02:56:38.637779+0000 mon.a (mon.0) 67 : audit 1 from='client.? 172.21.15.145:0/3935525350' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-03-07T02:56:39.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:39 smithi145 bash[21339]: cluster 2024-03-07T02:56:38.674784+0000 mgr.y (mgr.14120) 22 : cluster 0 pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:56:39.573 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:39 smithi145 bash[21623]: ignoring --setuser ceph since I am not root 2024-03-07T02:56:39.573 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:39 smithi145 bash[21623]: ignoring --setgroup ceph since I am not root 2024-03-07T02:56:39.573 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:39 smithi145 bash[21623]: debug 2024-03-07T02:56:39.341+0000 7fc9d3e01200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T02:56:40.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:40 smithi145 bash[21339]: audit 2024-03-07T02:56:39.152175+0000 mon.a (mon.0) 68 : audit 1 from='client.? 172.21.15.145:0/3935525350' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-03-07T02:56:40.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:40 smithi145 bash[21339]: cluster 2024-03-07T02:56:39.152279+0000 mon.a (mon.0) 69 : cluster 0 mgrmap e9: y(active, since 24s) 2024-03-07T02:56:41.269 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:40 smithi145 bash[21623]: debug 2024-03-07T02:56:40.917+0000 7fc9d3e01200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T02:56:41.269 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:41 smithi145 bash[21623]: debug 2024-03-07T02:56:41.017+0000 7fc9d3e01200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T02:56:41.269 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:41 smithi145 bash[21623]: debug 2024-03-07T02:56:41.265+0000 7fc9d3e01200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T02:56:41.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:41 smithi145 bash[21339]: audit 2024-03-07T02:56:41.277348+0000 mon.a (mon.0) 70 : audit 0 from='client.? 172.21.15.145:0/3336672157' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-07T02:56:41.918 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:56:41.918 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-03-07T02:56:41.918 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-07T02:56:41.918 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-03-07T02:56:41.918 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-03-07T02:56:41.918 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:56:41.918 INFO:teuthology.orchestra.run.smithi145.stdout:Waiting for the mgr to restart... 2024-03-07T02:56:41.918 INFO:teuthology.orchestra.run.smithi145.stdout:Waiting for mgr epoch 9... 2024-03-07T02:56:42.279 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:42 smithi145 bash[21623]: debug 2024-03-07T02:56:42.061+0000 7fc9d3e01200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T02:56:42.279 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:42 smithi145 bash[21623]: debug 2024-03-07T02:56:42.173+0000 7fc9d3e01200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T02:56:42.599 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:42 smithi145 bash[21623]: debug 2024-03-07T02:56:42.277+0000 7fc9d3e01200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T02:56:42.999 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:42 smithi145 bash[21623]: debug 2024-03-07T02:56:42.633+0000 7fc9d3e01200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T02:56:42.999 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:42 smithi145 bash[21623]: debug 2024-03-07T02:56:42.745+0000 7fc9d3e01200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T02:56:42.999 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:42 smithi145 bash[21623]: debug 2024-03-07T02:56:42.849+0000 7fc9d3e01200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T02:56:43.322 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:43 smithi145 bash[21623]: debug 2024-03-07T02:56:43.209+0000 7fc9d3e01200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T02:56:43.626 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:43 smithi145 bash[21623]: debug 2024-03-07T02:56:43.457+0000 7fc9d3e01200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T02:56:44.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:43 smithi145 bash[21623]: debug 2024-03-07T02:56:43.717+0000 7fc9d3e01200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T02:56:44.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:43 smithi145 bash[21623]: debug 2024-03-07T02:56:43.829+0000 7fc9d3e01200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T02:56:44.572 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:44 smithi145 bash[21623]: debug 2024-03-07T02:56:44.101+0000 7fc9d3e01200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T02:56:44.572 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:44 smithi145 bash[21623]: debug 2024-03-07T02:56:44.345+0000 7fc9d3e01200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:56:45.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:44 smithi145 bash[21623]: debug 2024-03-07T02:56:44.753+0000 7fc9d3e01200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T02:56:45.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:44 smithi145 bash[21623]: debug 2024-03-07T02:56:44.869+0000 7fc9d3e01200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T02:56:45.572 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:45 smithi145 bash[21623]: debug 2024-03-07T02:56:45.113+0000 7fc9d3e01200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:56:46.514 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:46 smithi145 bash[21623]: debug 2024-03-07T02:56:46.105+0000 7fc9d3e01200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T02:56:46.515 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:46 smithi145 bash[21623]: debug 2024-03-07T02:56:46.221+0000 7fc9d3e01200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T02:56:46.515 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:46 smithi145 bash[21623]: debug 2024-03-07T02:56:46.509+0000 7fc9d3e01200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T02:56:46.822 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:46 smithi145 bash[21623]: debug 2024-03-07T02:56:46.625+0000 7fc9d3e01200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T02:56:47.334 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:47 smithi145 bash[21623]: debug 2024-03-07T02:56:47.069+0000 7fc9d3e01200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T02:56:47.335 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:56:47 smithi145 bash[21623]: debug 2024-03-07T02:56:47.189+0000 7fc9d3e01200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T02:56:47.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:47 smithi145 bash[21339]: cluster 2024-03-07T02:56:47.212671+0000 mon.a (mon.0) 71 : cluster 1 Active manager daemon y restarted 2024-03-07T02:56:47.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:47 smithi145 bash[21339]: cluster 2024-03-07T02:56:47.213137+0000 mon.a (mon.0) 72 : cluster 1 Activating manager daemon y 2024-03-07T02:56:47.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:47 smithi145 bash[21339]: cluster 2024-03-07T02:56:47.220041+0000 mon.a (mon.0) 73 : cluster 0 osdmap e3: 0 total, 0 up, 0 in 2024-03-07T02:56:47.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:47 smithi145 bash[21339]: cluster 2024-03-07T02:56:47.220297+0000 mon.a (mon.0) 74 : cluster 0 mgrmap e10: y(active, starting, since 0.00735912s) 2024-03-07T02:56:47.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:47 smithi145 bash[21339]: audit 2024-03-07T02:56:47.221435+0000 mon.a (mon.0) 75 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T02:56:47.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:47 smithi145 bash[21339]: audit 2024-03-07T02:56:47.223286+0000 mon.a (mon.0) 76 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T02:56:47.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:47 smithi145 bash[21339]: audit 2024-03-07T02:56:47.224344+0000 mon.a (mon.0) 77 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T02:56:47.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:47 smithi145 bash[21339]: audit 2024-03-07T02:56:47.224499+0000 mon.a (mon.0) 78 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T02:56:47.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:47 smithi145 bash[21339]: audit 2024-03-07T02:56:47.224642+0000 mon.a (mon.0) 79 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T02:56:47.335 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:47 smithi145 bash[21339]: cluster 2024-03-07T02:56:47.248333+0000 mon.a (mon.0) 80 : cluster 1 Manager daemon y is now available 2024-03-07T02:56:48.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:48 smithi145 bash[21339]: audit 2024-03-07T02:56:47.267682+0000 mon.a (mon.0) 81 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:56:48.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:48 smithi145 bash[21339]: audit 2024-03-07T02:56:47.269052+0000 mon.a (mon.0) 82 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:56:48.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:48 smithi145 bash[21339]: audit 2024-03-07T02:56:47.287531+0000 mon.a (mon.0) 83 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T02:56:48.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:48 smithi145 bash[21339]: audit 2024-03-07T02:56:47.291135+0000 mon.a (mon.0) 84 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T02:56:48.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:48 smithi145 bash[21339]: cephadm 2024-03-07T02:56:48.110285+0000 mgr.y (mgr.14152) 1 : cephadm 1 [07/Mar/2024:02:56:48] ENGINE Bus STARTING 2024-03-07T02:56:48.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:48 smithi145 bash[21339]: cluster 2024-03-07T02:56:48.227398+0000 mon.a (mon.0) 85 : cluster 0 mgrmap e11: y(active, since 1.01447s) 2024-03-07T02:56:48.875 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:56:48.875 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-03-07T02:56:48.875 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout "initialized": true 2024-03-07T02:56:48.875 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:56:48.875 INFO:teuthology.orchestra.run.smithi145.stdout:mgr epoch 9 is available 2024-03-07T02:56:48.875 INFO:teuthology.orchestra.run.smithi145.stdout:Generating a dashboard self-signed certificate... 2024-03-07T02:56:49.532 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:49 smithi145 bash[21339]: audit 2024-03-07T02:56:48.225388+0000 mgr.y (mgr.14152) 2 : audit 0 from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-07T02:56:49.533 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:49 smithi145 bash[21339]: cephadm 2024-03-07T02:56:48.228126+0000 mgr.y (mgr.14152) 3 : cephadm 1 [07/Mar/2024:02:56:48] ENGINE Serving on https://172.21.15.145:7150 2024-03-07T02:56:49.533 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:49 smithi145 bash[21339]: audit 2024-03-07T02:56:48.237503+0000 mgr.y (mgr.14152) 4 : audit 0 from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-07T02:56:49.533 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:49 smithi145 bash[21339]: cephadm 2024-03-07T02:56:48.329763+0000 mgr.y (mgr.14152) 5 : cephadm 1 [07/Mar/2024:02:56:48] ENGINE Serving on http://172.21.15.145:8765 2024-03-07T02:56:49.533 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:49 smithi145 bash[21339]: cephadm 2024-03-07T02:56:48.329931+0000 mgr.y (mgr.14152) 6 : cephadm 1 [07/Mar/2024:02:56:48] ENGINE Bus STARTED 2024-03-07T02:56:50.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:50 smithi145 bash[21339]: cluster 2024-03-07T02:56:49.278308+0000 mon.a (mon.0) 86 : cluster 0 mgrmap e12: y(active, since 2s) 2024-03-07T02:56:50.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:50 smithi145 bash[21339]: audit 2024-03-07T02:56:50.117214+0000 mgr.y (mgr.14152) 7 : audit 0 from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:56:50.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:50 smithi145 bash[21339]: audit 2024-03-07T02:56:50.201698+0000 mon.a (mon.0) 87 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:56:50.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:50 smithi145 bash[21339]: audit 2024-03-07T02:56:50.207090+0000 mon.a (mon.0) 88 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:56:50.805 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-03-07T02:56:50.805 INFO:teuthology.orchestra.run.smithi145.stdout:Creating initial admin user... 2024-03-07T02:56:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:52 smithi145 bash[21339]: audit 2024-03-07T02:56:52.038609+0000 mgr.y (mgr.14152) 8 : audit 0 from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:56:52.902 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$kVSiSdU0cfzDeF7j0LkzUeQIGXsCY0B3PTS9UYBpaog9A55YydE2W", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1709780212, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-03-07T02:56:52.902 INFO:teuthology.orchestra.run.smithi145.stdout:Fetching dashboard port number... 2024-03-07T02:56:53.531 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:53 smithi145 bash[21339]: audit 2024-03-07T02:56:52.280357+0000 mon.a (mon.0) 89 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:56:54.699 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stdout 8443 2024-03-07T02:56:54.699 INFO:teuthology.orchestra.run.smithi145.stdout:firewalld does not appear to be present 2024-03-07T02:56:54.699 INFO:teuthology.orchestra.run.smithi145.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-03-07T02:56:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:Ceph Dashboard is now available at: 2024-03-07T02:56:54.703 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T02:56:54.703 INFO:teuthology.orchestra.run.smithi145.stdout: URL: https://smithi145.front.sepia.ceph.com:8443/ 2024-03-07T02:56:54.703 INFO:teuthology.orchestra.run.smithi145.stdout: User: admin 2024-03-07T02:56:54.703 INFO:teuthology.orchestra.run.smithi145.stdout: Password: mlzalsprgu 2024-03-07T02:56:54.703 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T02:56:54.703 INFO:teuthology.orchestra.run.smithi145.stdout:Saving cluster configuration to /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config directory 2024-03-07T02:56:54.704 INFO:teuthology.orchestra.run.smithi145.stdout:Enabling autotune for osd_memory_target 2024-03-07T02:56:55.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:55 smithi145 bash[21339]: audit 2024-03-07T02:56:54.029470+0000 mon.a (mon.0) 90 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:56:55.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:55 smithi145 bash[21339]: audit 2024-03-07T02:56:54.104764+0000 mon.a (mon.0) 91 : audit 0 from='client.? 172.21.15.145:0/2892840223' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-03-07T02:56:55.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:55 smithi145 bash[21339]: cluster 2024-03-07T02:56:54.144611+0000 mon.a (mon.0) 92 : cluster 0 mgrmap e13: y(active, since 6s) 2024-03-07T02:56:55.317 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:55 smithi145 bash[21339]: audit 2024-03-07T02:56:54.822910+0000 mon.a (mon.0) 93 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:56:58.587 INFO:teuthology.orchestra.run.smithi145.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-03-07T02:56:58.587 INFO:teuthology.orchestra.run.smithi145.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-03-07T02:56:58.587 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T02:56:58.587 INFO:teuthology.orchestra.run.smithi145.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-03-07T02:56:58.587 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T02:56:58.587 INFO:teuthology.orchestra.run.smithi145.stdout:Or, if you are only running a single cluster on this host: 2024-03-07T02:56:58.588 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T02:56:58.588 INFO:teuthology.orchestra.run.smithi145.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-03-07T02:56:58.588 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T02:56:58.588 INFO:teuthology.orchestra.run.smithi145.stdout:Please consider enabling telemetry to help improve Ceph: 2024-03-07T02:56:58.588 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T02:56:58.588 INFO:teuthology.orchestra.run.smithi145.stdout: ceph telemetry on 2024-03-07T02:56:58.588 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T02:56:58.588 INFO:teuthology.orchestra.run.smithi145.stdout:For more information see: 2024-03-07T02:56:58.588 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T02:56:58.588 INFO:teuthology.orchestra.run.smithi145.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-03-07T02:56:58.589 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T02:56:58.589 INFO:teuthology.orchestra.run.smithi145.stdout:Bootstrap complete. 2024-03-07T02:56:58.634 INFO:tasks.cephadm:Fetching config... 2024-03-07T02:56:58.634 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:56:58.634 DEBUG:teuthology.orchestra.run.smithi145:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-03-07T02:56:58.640 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-03-07T02:56:58.640 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:56:58.641 DEBUG:teuthology.orchestra.run.smithi145:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-03-07T02:56:58.687 INFO:tasks.cephadm:Fetching mon keyring... 2024-03-07T02:56:58.687 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:56:58.687 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/keyring of=/dev/stdout 2024-03-07T02:56:58.741 INFO:tasks.cephadm:Fetching pub ssh key... 2024-03-07T02:56:58.741 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:56:58.741 DEBUG:teuthology.orchestra.run.smithi145:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-03-07T02:56:58.786 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-03-07T02:56:58.787 DEBUG:teuthology.orchestra.run.smithi145:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDojO7HJbq4Kxb+/Tnl+CCWG5QEARyXEL/prNEEYqkUT1QBJASTMqWthwmets1IdPWFbxKzspwvrG7QSfSxV3b74SxAesK8V/6u+42PosIiBhfq2p1qpMKlGRgx2CaPKsXdGD4jQm1iWiTvyObi8TXQYyqu/jE8xp01mSgQk4cYhh/prWQ7vEZ1moEG0sQlRb86bAyt2U8w/5HLdI8T+Txek37nKrj7f0ff01AlIPRhaRQYzkebCxYTaYV80Le9LI/LYcyRsJYfThBZO6u0YR/5EC14kbqOkT9wCNL/VSB0eLLUDw+SipCWLWwvRTk2eG7ngnDxM7Z/y0i8UZdVVYwXdQMFdLXZl92lEP7o8nlFNqQqfdX+R/uVp22w8DrIy+jhwQZ2vwBoM3gNUhBSdI0veeqGXo+XZ+1n8UDBwpq99x/4mgwGBjDZmUnvqpa3k1z8CaulKzdp30R1FlX5wgd34RkyvUFWvuPXNo1n5ReXjW3PJjEn00SHRCLL0DZ9pR0= ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-07T02:56:58.848 INFO:teuthology.orchestra.run.smithi145.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDojO7HJbq4Kxb+/Tnl+CCWG5QEARyXEL/prNEEYqkUT1QBJASTMqWthwmets1IdPWFbxKzspwvrG7QSfSxV3b74SxAesK8V/6u+42PosIiBhfq2p1qpMKlGRgx2CaPKsXdGD4jQm1iWiTvyObi8TXQYyqu/jE8xp01mSgQk4cYhh/prWQ7vEZ1moEG0sQlRb86bAyt2U8w/5HLdI8T+Txek37nKrj7f0ff01AlIPRhaRQYzkebCxYTaYV80Le9LI/LYcyRsJYfThBZO6u0YR/5EC14kbqOkT9wCNL/VSB0eLLUDw+SipCWLWwvRTk2eG7ngnDxM7Z/y0i8UZdVVYwXdQMFdLXZl92lEP7o8nlFNqQqfdX+R/uVp22w8DrIy+jhwQZ2vwBoM3gNUhBSdI0veeqGXo+XZ+1n8UDBwpq99x/4mgwGBjDZmUnvqpa3k1z8CaulKzdp30R1FlX5wgd34RkyvUFWvuPXNo1n5ReXjW3PJjEn00SHRCLL0DZ9pR0= ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:56:58.856 DEBUG:teuthology.orchestra.run.smithi189:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDojO7HJbq4Kxb+/Tnl+CCWG5QEARyXEL/prNEEYqkUT1QBJASTMqWthwmets1IdPWFbxKzspwvrG7QSfSxV3b74SxAesK8V/6u+42PosIiBhfq2p1qpMKlGRgx2CaPKsXdGD4jQm1iWiTvyObi8TXQYyqu/jE8xp01mSgQk4cYhh/prWQ7vEZ1moEG0sQlRb86bAyt2U8w/5HLdI8T+Txek37nKrj7f0ff01AlIPRhaRQYzkebCxYTaYV80Le9LI/LYcyRsJYfThBZO6u0YR/5EC14kbqOkT9wCNL/VSB0eLLUDw+SipCWLWwvRTk2eG7ngnDxM7Z/y0i8UZdVVYwXdQMFdLXZl92lEP7o8nlFNqQqfdX+R/uVp22w8DrIy+jhwQZ2vwBoM3gNUhBSdI0veeqGXo+XZ+1n8UDBwpq99x/4mgwGBjDZmUnvqpa3k1z8CaulKzdp30R1FlX5wgd34RkyvUFWvuPXNo1n5ReXjW3PJjEn00SHRCLL0DZ9pR0= ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-07T02:56:58.881 INFO:teuthology.orchestra.run.smithi189.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDojO7HJbq4Kxb+/Tnl+CCWG5QEARyXEL/prNEEYqkUT1QBJASTMqWthwmets1IdPWFbxKzspwvrG7QSfSxV3b74SxAesK8V/6u+42PosIiBhfq2p1qpMKlGRgx2CaPKsXdGD4jQm1iWiTvyObi8TXQYyqu/jE8xp01mSgQk4cYhh/prWQ7vEZ1moEG0sQlRb86bAyt2U8w/5HLdI8T+Txek37nKrj7f0ff01AlIPRhaRQYzkebCxYTaYV80Le9LI/LYcyRsJYfThBZO6u0YR/5EC14kbqOkT9wCNL/VSB0eLLUDw+SipCWLWwvRTk2eG7ngnDxM7Z/y0i8UZdVVYwXdQMFdLXZl92lEP7o8nlFNqQqfdX+R/uVp22w8DrIy+jhwQZ2vwBoM3gNUhBSdI0veeqGXo+XZ+1n8UDBwpq99x/4mgwGBjDZmUnvqpa3k1z8CaulKzdp30R1FlX5wgd34RkyvUFWvuPXNo1n5ReXjW3PJjEn00SHRCLL0DZ9pR0= ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:56:58.891 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-03-07T02:56:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:56:58 smithi145 bash[21339]: audit 2024-03-07T02:56:57.977711+0000 mon.a (mon.0) 94 : audit 1 from='client.? 172.21.15.145:0/2285641272' entity='client.admin' 2024-03-07T02:57:03.868 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T02:57:07.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:07 smithi145 bash[21339]: audit 2024-03-07T02:57:06.221045+0000 mon.a (mon.0) 95 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:07.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:07 smithi145 bash[21339]: audit 2024-03-07T02:57:06.227692+0000 mon.a (mon.0) 96 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:07.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:07 smithi145 bash[21339]: audit 2024-03-07T02:57:06.229547+0000 mon.a (mon.0) 97 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi145", "name": "osd_memory_target"}]: dispatch 2024-03-07T02:57:07.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:07 smithi145 bash[21339]: audit 2024-03-07T02:57:06.236640+0000 mon.a (mon.0) 98 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:07.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:07 smithi145 bash[21339]: audit 2024-03-07T02:57:06.271652+0000 mon.a (mon.0) 99 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:57:07.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:07 smithi145 bash[21339]: audit 2024-03-07T02:57:06.280257+0000 mon.a (mon.0) 100 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:07.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:07 smithi145 bash[21339]: audit 2024-03-07T02:57:06.740875+0000 mon.a (mon.0) 101 : audit 1 from='client.? 172.21.15.145:0/3864232839' entity='client.admin' 2024-03-07T02:57:07.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:07 smithi145 bash[21339]: audit 2024-03-07T02:57:06.778207+0000 mon.a (mon.0) 102 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:57:07.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:07 smithi145 bash[21339]: audit 2024-03-07T02:57:06.783929+0000 mon.a (mon.0) 103 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:07.533 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-03-07T02:57:07.533 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-03-07T02:57:09.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:09 smithi145 bash[21339]: cluster 2024-03-07T02:57:07.226038+0000 mgr.y (mgr.14152) 9 : cluster 0 pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:11.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:11 smithi145 bash[21339]: cluster 2024-03-07T02:57:09.226462+0000 mgr.y (mgr.14152) 10 : cluster 0 pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:12.196 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T02:57:13.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:13 smithi145 bash[21339]: cluster 2024-03-07T02:57:11.226825+0000 mgr.y (mgr.14152) 11 : cluster 0 pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:15.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:15 smithi145 bash[21339]: cluster 2024-03-07T02:57:13.227212+0000 mgr.y (mgr.14152) 12 : cluster 0 pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:15.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:15 smithi145 bash[21339]: audit 2024-03-07T02:57:14.952721+0000 mon.a (mon.0) 104 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:15.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:15 smithi145 bash[21339]: audit 2024-03-07T02:57:14.985023+0000 mon.a (mon.0) 105 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:57:15.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:15 smithi145 bash[21339]: audit 2024-03-07T02:57:14.986367+0000 mon.a (mon.0) 106 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:57:15.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:15 smithi145 bash[21339]: audit 2024-03-07T02:57:14.987030+0000 mon.a (mon.0) 107 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:57:15.682 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi189 2024-03-07T02:57:15.682 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-07T02:57:15.682 DEBUG:teuthology.orchestra.run.smithi189:> dd of=/etc/ceph/ceph.conf 2024-03-07T02:57:15.689 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-07T02:57:15.689 DEBUG:teuthology.orchestra.run.smithi189:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-07T02:57:15.736 INFO:tasks.cephadm:Adding host smithi189 to orchestrator... 2024-03-07T02:57:15.737 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch host add smithi189 2024-03-07T02:57:16.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:16 smithi145 bash[21339]: audit 2024-03-07T02:57:14.947641+0000 mgr.y (mgr.14152) 13 : audit 0 from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:57:16.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:16 smithi145 bash[21339]: cephadm 2024-03-07T02:57:14.987910+0000 mgr.y (mgr.14152) 14 : cephadm 1 Updating smithi145:/etc/ceph/ceph.conf 2024-03-07T02:57:16.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:16 smithi145 bash[21339]: cephadm 2024-03-07T02:57:15.101903+0000 mgr.y (mgr.14152) 15 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T02:57:16.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:16 smithi145 bash[21339]: cephadm 2024-03-07T02:57:15.215602+0000 mgr.y (mgr.14152) 16 : cephadm 1 Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-03-07T02:57:16.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:16 smithi145 bash[21339]: audit 2024-03-07T02:57:15.599392+0000 mon.a (mon.0) 108 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:16.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:16 smithi145 bash[21339]: audit 2024-03-07T02:57:15.603687+0000 mon.a (mon.0) 109 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:16.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:16 smithi145 bash[21339]: audit 2024-03-07T02:57:15.607899+0000 mon.a (mon.0) 110 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:17.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:17 smithi145 bash[21339]: cluster 2024-03-07T02:57:15.227531+0000 mgr.y (mgr.14152) 17 : cluster 0 pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:17.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:17 smithi145 bash[21339]: cephadm 2024-03-07T02:57:15.380354+0000 mgr.y (mgr.14152) 18 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T02:57:19.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:19 smithi145 bash[21339]: cluster 2024-03-07T02:57:17.227892+0000 mgr.y (mgr.14152) 19 : cluster 0 pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:20.400 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T02:57:21.519 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:21 smithi145 bash[21339]: cluster 2024-03-07T02:57:19.228298+0000 mgr.y (mgr.14152) 20 : cluster 0 pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:23.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:23 smithi145 bash[21339]: cluster 2024-03-07T02:57:21.228577+0000 mgr.y (mgr.14152) 21 : cluster 0 pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:24.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:24 smithi145 bash[21339]: audit 2024-03-07T02:57:23.056940+0000 mgr.y (mgr.14152) 22 : audit 0 from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi189", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:57:25.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:25 smithi145 bash[21339]: cluster 2024-03-07T02:57:23.229004+0000 mgr.y (mgr.14152) 23 : cluster 0 pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:25.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:25 smithi145 bash[21339]: cephadm 2024-03-07T02:57:23.803823+0000 mgr.y (mgr.14152) 24 : cephadm 1 Deploying cephadm binary to smithi189 2024-03-07T02:57:26.383 INFO:teuthology.orchestra.run.smithi145.stdout:Added host 'smithi189' with addr '172.21.15.189' 2024-03-07T02:57:27.051 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch host ls --format=json 2024-03-07T02:57:27.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:27 smithi145 bash[21339]: cluster 2024-03-07T02:57:25.229463+0000 mgr.y (mgr.14152) 25 : cluster 0 pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:27.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:27 smithi145 bash[21339]: audit 2024-03-07T02:57:26.381302+0000 mon.a (mon.0) 111 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:27.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:27 smithi145 bash[21339]: audit 2024-03-07T02:57:26.432370+0000 mon.a (mon.0) 112 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:57:27.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:27 smithi145 bash[21339]: audit 2024-03-07T02:57:26.754729+0000 mon.a (mon.0) 113 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:28.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:28 smithi145 bash[21339]: cephadm 2024-03-07T02:57:26.382098+0000 mgr.y (mgr.14152) 26 : cephadm 1 Added host smithi189 2024-03-07T02:57:28.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:28 smithi145 bash[21339]: audit 2024-03-07T02:57:28.128978+0000 mon.a (mon.0) 114 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:29.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:29 smithi145 bash[21339]: cluster 2024-03-07T02:57:27.229776+0000 mgr.y (mgr.14152) 27 : cluster 0 pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:29.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:29 smithi145 bash[21339]: audit 2024-03-07T02:57:29.124922+0000 mon.a (mon.0) 115 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:31.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:31 smithi145 bash[21339]: cluster 2024-03-07T02:57:29.230158+0000 mgr.y (mgr.14152) 28 : cluster 0 pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:31.720 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T02:57:33.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:33 smithi145 bash[21339]: cluster 2024-03-07T02:57:31.230451+0000 mgr.y (mgr.14152) 29 : cluster 0 pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:34.471 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T02:57:34.471 INFO:teuthology.orchestra.run.smithi145.stdout:[{"addr": "172.21.15.145", "hostname": "smithi145", "labels": [], "status": ""}, {"addr": "172.21.15.189", "hostname": "smithi189", "labels": [], "status": ""}] 2024-03-07T02:57:35.168 INFO:tasks.cephadm:Setting crush tunables to default 2024-03-07T02:57:35.168 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd crush tunables default 2024-03-07T02:57:35.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:35 smithi145 bash[21339]: cluster 2024-03-07T02:57:33.230815+0000 mgr.y (mgr.14152) 30 : cluster 0 pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:35.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:35 smithi145 bash[21339]: audit 2024-03-07T02:57:34.062790+0000 mon.a (mon.0) 116 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:35.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:35 smithi145 bash[21339]: audit 2024-03-07T02:57:34.068980+0000 mon.a (mon.0) 117 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:35.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:35 smithi145 bash[21339]: audit 2024-03-07T02:57:34.075611+0000 mon.a (mon.0) 118 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:35.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:35 smithi145 bash[21339]: audit 2024-03-07T02:57:34.082386+0000 mon.a (mon.0) 119 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:35.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:35 smithi145 bash[21339]: audit 2024-03-07T02:57:34.083337+0000 mon.a (mon.0) 120 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi189", "name": "osd_memory_target"}]: dispatch 2024-03-07T02:57:35.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:35 smithi145 bash[21339]: audit 2024-03-07T02:57:34.084314+0000 mon.a (mon.0) 121 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:57:35.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:35 smithi145 bash[21339]: audit 2024-03-07T02:57:34.085131+0000 mon.a (mon.0) 122 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:57:35.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:35 smithi145 bash[21339]: cephadm 2024-03-07T02:57:34.086397+0000 mgr.y (mgr.14152) 31 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T02:57:35.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:35 smithi145 bash[21339]: cephadm 2024-03-07T02:57:34.200676+0000 mgr.y (mgr.14152) 32 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T02:57:35.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:35 smithi145 bash[21339]: audit 2024-03-07T02:57:34.652016+0000 mon.a (mon.0) 123 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:35.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:35 smithi145 bash[21339]: audit 2024-03-07T02:57:34.657009+0000 mon.a (mon.0) 124 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:35.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:35 smithi145 bash[21339]: audit 2024-03-07T02:57:34.662303+0000 mon.a (mon.0) 125 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:36.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:36 smithi145 bash[21339]: cephadm 2024-03-07T02:57:34.365651+0000 mgr.y (mgr.14152) 33 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T02:57:36.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:36 smithi145 bash[21339]: audit 2024-03-07T02:57:34.470590+0000 mgr.y (mgr.14152) 34 : audit 0 from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T02:57:36.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:36 smithi145 bash[21339]: cephadm 2024-03-07T02:57:34.530370+0000 mgr.y (mgr.14152) 35 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T02:57:37.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:37 smithi145 bash[21339]: cluster 2024-03-07T02:57:35.231160+0000 mgr.y (mgr.14152) 36 : cluster 0 pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:39.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:39 smithi145 bash[21339]: cluster 2024-03-07T02:57:37.231540+0000 mgr.y (mgr.14152) 37 : cluster 0 pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:39.836 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T02:57:41.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:41 smithi145 bash[21339]: cluster 2024-03-07T02:57:39.231951+0000 mgr.y (mgr.14152) 38 : cluster 0 pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:43.168 INFO:teuthology.orchestra.run.smithi145.stderr:adjusted tunables profile to default 2024-03-07T02:57:43.415 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:43 smithi145 bash[21339]: cluster 2024-03-07T02:57:41.232366+0000 mgr.y (mgr.14152) 39 : cluster 0 pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:43.415 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:43 smithi145 bash[21339]: audit 2024-03-07T02:57:42.648462+0000 mon.a (mon.0) 126 : audit 1 from='client.? 172.21.15.145:0/1996668903' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-07T02:57:43.862 INFO:tasks.cephadm:Adding mon.a on smithi145 2024-03-07T02:57:43.862 INFO:tasks.cephadm:Adding mon.c on smithi145 2024-03-07T02:57:43.862 INFO:tasks.cephadm:Adding mon.b on smithi189 2024-03-07T02:57:43.863 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch apply mon '3;smithi145:172.21.15.145=a;smithi145:[v2:172.21.15.145:3301,v1:172.21.15.145:6790]=c;smithi189:172.21.15.189=b' 2024-03-07T02:57:44.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:44 smithi145 bash[21339]: audit 2024-03-07T02:57:43.162481+0000 mon.a (mon.0) 127 : audit 1 from='client.? 172.21.15.145:0/1996668903' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-07T02:57:44.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:44 smithi145 bash[21339]: cluster 2024-03-07T02:57:43.162653+0000 mon.a (mon.0) 128 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T02:57:45.027 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T02:57:45.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:45 smithi145 bash[21339]: cluster 2024-03-07T02:57:43.232674+0000 mgr.y (mgr.14152) 40 : cluster 0 pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:47.419 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled mon update... 2024-03-07T02:57:47.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:47 smithi145 bash[21339]: cluster 2024-03-07T02:57:45.233049+0000 mgr.y (mgr.14152) 41 : cluster 0 pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:48.245 DEBUG:teuthology.orchestra.run.smithi145:mon.c> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.c.service 2024-03-07T02:57:48.249 DEBUG:teuthology.orchestra.run.smithi189:mon.b> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.b.service 2024-03-07T02:57:48.252 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-07T02:57:48.252 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph mon dump -f json 2024-03-07T02:57:48.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:48 smithi145 bash[21339]: cluster 2024-03-07T02:57:47.233435+0000 mgr.y (mgr.14152) 42 : cluster 0 pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:48.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:48 smithi145 bash[21339]: audit 2024-03-07T02:57:47.406456+0000 mgr.y (mgr.14152) 43 : audit 0 from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi145:172.21.15.145=a;smithi145:[v2:172.21.15.145:3301,v1:172.21.15.145:6790]=c;smithi189:172.21.15.189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:57:48.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:48 smithi145 bash[21339]: cephadm 2024-03-07T02:57:47.412389+0000 mgr.y (mgr.14152) 44 : cephadm 1 Saving service mon spec with placement smithi145:172.21.15.145=a;smithi145:[v2:172.21.15.145:3301,v1:172.21.15.145:6790]=c;smithi189:172.21.15.189=b;count:3 2024-03-07T02:57:48.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:48 smithi145 bash[21339]: audit 2024-03-07T02:57:47.418094+0000 mon.a (mon.0) 129 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:48.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:48 smithi145 bash[21339]: audit 2024-03-07T02:57:47.423893+0000 mon.a (mon.0) 130 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:57:48.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:48 smithi145 bash[21339]: audit 2024-03-07T02:57:47.427638+0000 mon.a (mon.0) 131 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:57:48.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:48 smithi145 bash[21339]: audit 2024-03-07T02:57:47.429296+0000 mon.a (mon.0) 132 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:57:48.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:48 smithi145 bash[21339]: audit 2024-03-07T02:57:47.437279+0000 mon.a (mon.0) 133 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:48.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:48 smithi145 bash[21339]: audit 2024-03-07T02:57:47.441300+0000 mon.a (mon.0) 134 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T02:57:48.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:48 smithi145 bash[21339]: audit 2024-03-07T02:57:47.443073+0000 mon.a (mon.0) 135 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:57:48.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:48 smithi145 bash[21339]: cephadm 2024-03-07T02:57:47.445047+0000 mgr.y (mgr.14152) 45 : cephadm 1 Deploying daemon mon.b on smithi189 2024-03-07T02:57:49.748 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T02:57:50.821 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:50 smithi145 bash[21339]: cluster 2024-03-07T02:57:49.233856+0000 mgr.y (mgr.14152) 46 : cluster 0 pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:52.537 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:52 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:57:52.821 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:52 smithi145 bash[21339]: cluster 2024-03-07T02:57:51.234262+0000 mgr.y (mgr.14152) 47 : cluster 0 pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:52.840 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:52 smithi189 systemd[1]: Started Ceph mon.b for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T02:57:52.951 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-07T02:57:52.951 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"8947a0f8-dc2d-11ee-95c6-87774f69a715","modified":"2024-03-07T02:55:37.551545Z","created":"2024-03-07T02:55:37.551545Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:3300","nonce":0},{"type":"v1","addr":"172.21.15.145:6789","nonce":0}]},"addr":"172.21.15.145:6789/0","public_addr":"172.21.15.145:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-07T02:57:52.953 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-03-07T02:57:53.841 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.775+0000 7f5f29196c80 0 set uid:gid to 167:167 (ceph:ceph) 2024-03-07T02:57:53.841 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.775+0000 7f5f29196c80 0 ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable), process ceph-mon, pid 7 2024-03-07T02:57:53.841 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.775+0000 7f5f29196c80 0 pidfile_write: ignore empty --pid-file 2024-03-07T02:57:53.841 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.779+0000 7f5f29196c80 0 load: jerasure load: lrc 2024-03-07T02:57:53.841 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: RocksDB version: 7.9.2 2024-03-07T02:57:53.841 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Git sha 0 2024-03-07T02:57:53.841 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Compile date 2024-03-06 23:19:49 2024-03-07T02:57:53.841 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: DB SUMMARY 2024-03-07T02:57:53.842 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: DB Session ID: X5J0REOMXUHK4H5JS0QX 2024-03-07T02:57:53.842 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: CURRENT file: CURRENT 2024-03-07T02:57:53.842 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: IDENTITY file: IDENTITY 2024-03-07T02:57:53.842 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-03-07T02:57:53.842 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-03-07T02:57:53.842 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-03-07T02:57:53.842 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.error_if_exists: 0 2024-03-07T02:57:53.842 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.create_if_missing: 0 2024-03-07T02:57:53.842 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.paranoid_checks: 1 2024-03-07T02:57:53.842 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-03-07T02:57:53.842 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-07T02:57:53.843 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-07T02:57:53.843 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.env: 0x55b457c3c720 2024-03-07T02:57:53.843 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.fs: PosixFileSystem 2024-03-07T02:57:53.843 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.info_log: 0x55b459315500 2024-03-07T02:57:53.843 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_file_opening_threads: 16 2024-03-07T02:57:53.843 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.statistics: (nil) 2024-03-07T02:57:53.843 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.use_fsync: 0 2024-03-07T02:57:53.843 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_log_file_size: 0 2024-03-07T02:57:53.843 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-07T02:57:53.843 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-03-07T02:57:53.843 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.keep_log_file_num: 1000 2024-03-07T02:57:53.843 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.recycle_log_file_num: 0 2024-03-07T02:57:53.844 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.allow_fallocate: 1 2024-03-07T02:57:53.844 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.allow_mmap_reads: 0 2024-03-07T02:57:53.844 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.allow_mmap_writes: 0 2024-03-07T02:57:53.844 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.use_direct_reads: 0 2024-03-07T02:57:53.844 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-07T02:57:53.844 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.create_missing_column_families: 0 2024-03-07T02:57:53.844 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.db_log_dir: 2024-03-07T02:57:53.844 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.wal_dir: 2024-03-07T02:57:53.844 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-03-07T02:57:53.844 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-03-07T02:57:53.845 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-03-07T02:57:53.845 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-07T02:57:53.845 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-07T02:57:53.845 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-03-07T02:57:53.845 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.advise_random_on_open: 1 2024-03-07T02:57:53.845 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.db_write_buffer_size: 0 2024-03-07T02:57:53.845 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.write_buffer_manager: 0x55b4585a2320 2024-03-07T02:57:53.845 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-07T02:57:53.845 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-07T02:57:53.845 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-03-07T02:57:53.845 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.rate_limiter: (nil) 2024-03-07T02:57:53.845 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-07T02:57:53.845 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.wal_recovery_mode: 2 2024-03-07T02:57:53.847 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.enable_thread_tracking: 0 2024-03-07T02:57:53.847 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.enable_pipelined_write: 0 2024-03-07T02:57:53.847 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.unordered_write: 0 2024-03-07T02:57:53.847 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-07T02:57:53.847 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-07T02:57:53.848 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-07T02:57:53.848 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-07T02:57:53.848 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.row_cache: None 2024-03-07T02:57:53.848 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.wal_filter: None 2024-03-07T02:57:53.848 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-07T02:57:53.848 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.allow_ingest_behind: 0 2024-03-07T02:57:53.848 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.two_write_queues: 0 2024-03-07T02:57:53.848 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.manual_wal_flush: 0 2024-03-07T02:57:53.848 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.wal_compression: 0 2024-03-07T02:57:53.848 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.atomic_flush: 0 2024-03-07T02:57:53.848 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-07T02:57:53.848 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-03-07T02:57:53.848 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.log_readahead_size: 0 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.best_efforts_recovery: 0 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.allow_data_in_errors: 0 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.db_host_id: __hostname__ 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_background_jobs: 2 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_background_compactions: -1 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_subcompactions: 1 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-07T02:57:53.849 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_total_wal_size: 0 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_open_files: -1 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.bytes_per_sync: 0 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compaction_readahead_size: 0 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_background_flushes: -1 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Compression algorithms supported: 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: kZSTD supported: 0 2024-03-07T02:57:53.850 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: kXpressCompression supported: 0 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: kLZ4HCCompression supported: 1 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: kZlibCompression supported: 1 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: kSnappyCompression supported: 1 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: kLZ4Compression supported: 1 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: kBZip2Compression supported: 0 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.merge_operator: 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compaction_filter: None 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compaction_filter_factory: None 2024-03-07T02:57:53.851 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.sst_partitioner_factory: None 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b459315600) 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cache_index_and_filter_blocks: 1 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: pin_top_level_index_and_filter: 1 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: index_type: 0 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: data_block_index_type: 0 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: index_shortening: 1 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: data_block_hash_table_util_ratio: 0.750000 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: checksum: 4 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: no_block_cache: 0 2024-03-07T02:57:53.852 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: block_cache: 0x55b4586251f0 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: block_cache_name: BinnedLRUCache 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: block_cache_options: 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: capacity : 536870912 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: num_shard_bits : 4 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: strict_capacity_limit : 0 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: high_pri_pool_ratio: 0.000 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: block_cache_compressed: (nil) 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: persistent_cache: (nil) 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: block_size: 4096 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: block_size_deviation: 10 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: block_restart_interval: 16 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: index_block_restart_interval: 1 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: metadata_block_size: 4096 2024-03-07T02:57:53.853 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: partition_filters: 0 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: use_delta_encoding: 1 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: filter_policy: bloomfilter 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: whole_key_filtering: 1 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: verify_compression: 0 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: read_amp_bytes_per_bit: 0 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: format_version: 5 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: enable_index_compression: 1 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: block_align: 0 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: max_auto_readahead_size: 262144 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: prepopulate_block_cache: 0 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: initial_auto_readahead_size: 8192 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: num_file_reads_for_auto_readahead: 2 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.write_buffer_size: 33554432 2024-03-07T02:57:53.854 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_write_buffer_number: 2 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compression: NoCompression 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.bottommost_compression: Disabled 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.prefix_extractor: nullptr 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.num_levels: 7 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-07T02:57:53.855 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-07T02:57:53.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-07T02:57:53.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-03-07T02:57:53.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compression_opts.level: 32767 2024-03-07T02:57:53.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compression_opts.strategy: 0 2024-03-07T02:57:53.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-07T02:57:53.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-07T02:57:53.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-07T02:57:53.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-07T02:57:53.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compression_opts.enabled: false 2024-03-07T02:57:53.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-07T02:57:53.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-07T02:57:53.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-07T02:57:53.856 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.target_file_size_base: 67108864 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-07T02:57:53.857 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.arena_block_size: 1048576 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.disable_auto_compactions: 0 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-07T02:57:53.858 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.table_properties_collectors: 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.inplace_update_support: 0 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.bloom_locality: 0 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.max_successive_merges: 0 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.paranoid_file_checks: 0 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.force_consistency_checks: 1 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.report_bg_io_stats: 0 2024-03-07T02:57:53.859 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.ttl: 2592000 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.enable_blob_files: false 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.min_blob_size: 0 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.blob_file_size: 268435456 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.blob_file_starting_level: 0 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-07T02:57:53.860 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 9aaa9f57-eed6-4d51-b5b7-c1373ff5ef78 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709780273787075, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709780273787550, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1622, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1709780273, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "9aaa9f57-eed6-4d51-b5b7-c1373ff5ef78", "db_session_id": "X5J0REOMXUHK4H5JS0QX", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709780273787667, "job": 1, "event": "recovery_finished"} 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.783+0000 7f5f29196c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.787+0000 7f5f29196c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.787+0000 7f5f29196c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b4586c6000 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.787+0000 7f5f29196c80 4 rocksdb: DB pointer 0x55b4586b4000 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.787+0000 7f5f29196c80 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.787+0000 7f5f29196c80 0 using public_addr v2:172.21.15.189:0/0 -> [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.787+0000 7f5f29196c80 0 starting mon.b rank -1 at public addrs [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] at bind addrs [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.787+0000 7f5f17990700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-07T02:57:53.861 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.787+0000 7f5f17990700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: ** DB Stats ** 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: ** Compaction Stats [default] ** 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: L0 1/0 1.58 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Sum 1/0 1.58 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: ** Compaction Stats [default] ** 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T02:57:53.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.5 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Flush(GB): cumulative 0.000, interval 0.000 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: AddFile(Total Files): cumulative 0, interval 0 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: AddFile(L0 Files): cumulative 0, interval 0 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: AddFile(Keys): cumulative 0, interval 0 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Cumulative compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Interval compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Block cache BinnedLRUCache@0x55b4586251f0#7 capacity: 512.00 MB usage: 0.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: Block cache entry stats(count,size,portion): DataBlock(1,0.64 KB,0.00012219%) FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-03-07T02:57:53.863 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: ** File Read Latency Histogram By Level [default] ** 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.787+0000 7f5f29196c80 1 mon.b@-1(???) e0 preinit fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.811+0000 7f5f1a996700 0 mon.b@-1(synchronizing).mds e1 new map 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.811+0000 7f5f1a996700 0 mon.b@-1(synchronizing).mds e1 print_map 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: e1 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: enable_multiple, ever_enabled_multiple: 1,1 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: legacy client fscid: -1 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: No filesystems configured 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.811+0000 7f5f1a996700 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.811+0000 7f5f1a996700 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.811+0000 7f5f1a996700 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.811+0000 7f5f1a996700 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.811+0000 7f5f1a996700 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-03-07T02:57:53.864 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.811+0000 7f5f1a996700 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.815+0000 7f5f1a996700 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.815+0000 7f5f1a996700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.815+0000 7f5f1a996700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.815+0000 7f5f1a996700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:21.228577+0000 mgr.y (mgr.14152) 21 : cluster 0 pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:23.056940+0000 mgr.y (mgr.14152) 22 : audit 0 from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi189", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:23.229004+0000 mgr.y (mgr.14152) 23 : cluster 0 pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cephadm 2024-03-07T02:57:23.803823+0000 mgr.y (mgr.14152) 24 : cephadm 1 Deploying cephadm binary to smithi189 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:25.229463+0000 mgr.y (mgr.14152) 25 : cluster 0 pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:26.381302+0000 mon.a (mon.0) 111 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:26.432370+0000 mon.a (mon.0) 112 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:26.754729+0000 mon.a (mon.0) 113 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cephadm 2024-03-07T02:57:26.382098+0000 mgr.y (mgr.14152) 26 : cephadm 1 Added host smithi189 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:28.128978+0000 mon.a (mon.0) 114 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:27.229776+0000 mgr.y (mgr.14152) 27 : cluster 0 pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:29.124922+0000 mon.a (mon.0) 115 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.865 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:29.230158+0000 mgr.y (mgr.14152) 28 : cluster 0 pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:31.230451+0000 mgr.y (mgr.14152) 29 : cluster 0 pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:33.230815+0000 mgr.y (mgr.14152) 30 : cluster 0 pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:34.062790+0000 mon.a (mon.0) 116 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:34.068980+0000 mon.a (mon.0) 117 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:34.075611+0000 mon.a (mon.0) 118 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:34.082386+0000 mon.a (mon.0) 119 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:34.083337+0000 mon.a (mon.0) 120 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi189", "name": "osd_memory_target"}]: dispatch 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:34.084314+0000 mon.a (mon.0) 121 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:34.085131+0000 mon.a (mon.0) 122 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cephadm 2024-03-07T02:57:34.086397+0000 mgr.y (mgr.14152) 31 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cephadm 2024-03-07T02:57:34.200676+0000 mgr.y (mgr.14152) 32 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:34.652016+0000 mon.a (mon.0) 123 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:34.657009+0000 mon.a (mon.0) 124 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:34.662303+0000 mon.a (mon.0) 125 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cephadm 2024-03-07T02:57:34.365651+0000 mgr.y (mgr.14152) 33 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T02:57:53.866 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:34.470590+0000 mgr.y (mgr.14152) 34 : audit 0 from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cephadm 2024-03-07T02:57:34.530370+0000 mgr.y (mgr.14152) 35 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:35.231160+0000 mgr.y (mgr.14152) 36 : cluster 0 pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:37.231540+0000 mgr.y (mgr.14152) 37 : cluster 0 pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:39.231951+0000 mgr.y (mgr.14152) 38 : cluster 0 pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:41.232366+0000 mgr.y (mgr.14152) 39 : cluster 0 pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:42.648462+0000 mon.a (mon.0) 126 : audit 1 from='client.? 172.21.15.145:0/1996668903' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:43.162481+0000 mon.a (mon.0) 127 : audit 1 from='client.? 172.21.15.145:0/1996668903' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:43.162653+0000 mon.a (mon.0) 128 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:43.232674+0000 mgr.y (mgr.14152) 40 : cluster 0 pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:45.233049+0000 mgr.y (mgr.14152) 41 : cluster 0 pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:47.233435+0000 mgr.y (mgr.14152) 42 : cluster 0 pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:47.406456+0000 mgr.y (mgr.14152) 43 : audit 0 from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi145:172.21.15.145=a;smithi145:[v2:172.21.15.145:3301,v1:172.21.15.145:6790]=c;smithi189:172.21.15.189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cephadm 2024-03-07T02:57:47.412389+0000 mgr.y (mgr.14152) 44 : cephadm 1 Saving service mon spec with placement smithi145:172.21.15.145=a;smithi145:[v2:172.21.15.145:3301,v1:172.21.15.145:6790]=c;smithi189:172.21.15.189=b;count:3 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:47.418094+0000 mon.a (mon.0) 129 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:47.423893+0000 mon.a (mon.0) 130 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:57:53.867 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:47.427638+0000 mon.a (mon.0) 131 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:47.429296+0000 mon.a (mon.0) 132 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:47.437279+0000 mon.a (mon.0) 133 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:47.441300+0000 mon.a (mon.0) 134 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:47.443073+0000 mon.a (mon.0) 135 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cephadm 2024-03-07T02:57:47.445047+0000 mgr.y (mgr.14152) 45 : cephadm 1 Deploying daemon mon.b on smithi189 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:49.233856+0000 mgr.y (mgr.14152) 46 : cluster 0 pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cluster 2024-03-07T02:57:51.234262+0000 mgr.y (mgr.14152) 47 : cluster 0 pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:52.576578+0000 mon.a (mon.0) 136 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:52.582516+0000 mon.a (mon.0) 137 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:52.627511+0000 mon.a (mon.0) 138 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:52.628848+0000 mon.a (mon.0) 139 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:52.630388+0000 mon.a (mon.0) 140 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: cephadm 2024-03-07T02:57:52.632480+0000 mgr.y (mgr.14152) 48 : cephadm 1 Deploying daemon mon.c on smithi145 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: audit 2024-03-07T02:57:52.950444+0000 mon.a (mon.0) 141 : audit 0 from='client.? 172.21.15.189:0/1330652387' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-07T02:57:53.868 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:53 smithi189 bash[23614]: debug 2024-03-07T02:57:53.815+0000 7f5f1a996700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-03-07T02:57:53.896 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:53 smithi145 bash[21339]: audit 2024-03-07T02:57:52.576578+0000 mon.a (mon.0) 136 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.896 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:53 smithi145 bash[21339]: audit 2024-03-07T02:57:52.582516+0000 mon.a (mon.0) 137 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.896 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:53 smithi145 bash[21339]: audit 2024-03-07T02:57:52.627511+0000 mon.a (mon.0) 138 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:53.896 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:53 smithi145 bash[21339]: audit 2024-03-07T02:57:52.628848+0000 mon.a (mon.0) 139 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T02:57:53.896 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:53 smithi145 bash[21339]: audit 2024-03-07T02:57:52.630388+0000 mon.a (mon.0) 140 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:57:53.896 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:53 smithi145 bash[21339]: cephadm 2024-03-07T02:57:52.632480+0000 mgr.y (mgr.14152) 48 : cephadm 1 Deploying daemon mon.c on smithi145 2024-03-07T02:57:53.896 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:53 smithi145 bash[21339]: audit 2024-03-07T02:57:52.950444+0000 mon.a (mon.0) 141 : audit 0 from='client.? 172.21.15.189:0/1330652387' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-07T02:57:54.675 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-07T02:57:54.676 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph mon dump -f json 2024-03-07T02:57:56.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:56 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:57:56.571 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:57:56 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:57:57.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:56 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:57:57.072 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:57:56 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:57:57.822 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:57:57 smithi145 bash[30906]: debug 2024-03-07T02:57:57.680+0000 7faa9f883700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-03-07T02:57:58.329 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T02:57:59.227 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: cluster 2024-03-07T02:57:53.234660+0000 mgr.y (mgr.14152) 49 : cluster 0 pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:59.227 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: audit 2024-03-07T02:57:53.827894+0000 mon.a (mon.0) 143 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: cluster 2024-03-07T02:57:53.828237+0000 mon.a (mon.0) 144 : cluster 1 mon.a calling monitor election 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: audit 2024-03-07T02:57:53.851274+0000 mon.a (mon.0) 145 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: audit 2024-03-07T02:57:54.822757+0000 mon.a (mon.0) 146 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: cluster 2024-03-07T02:57:55.235096+0000 mgr.y (mgr.14152) 50 : cluster 0 pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: audit 2024-03-07T02:57:55.823112+0000 mon.a (mon.0) 147 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: cluster 2024-03-07T02:57:55.826124+0000 mon.b (mon.1) 1 : cluster 1 mon.b calling monitor election 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: audit 2024-03-07T02:57:56.823702+0000 mon.a (mon.0) 148 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: cluster 2024-03-07T02:57:57.235517+0000 mgr.y (mgr.14152) 51 : cluster 0 pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: audit 2024-03-07T02:57:57.688277+0000 mon.a (mon.0) 149 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: audit 2024-03-07T02:57:57.823668+0000 mon.a (mon.0) 150 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: audit 2024-03-07T02:57:58.689153+0000 mon.a (mon.0) 151 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: audit 2024-03-07T02:57:58.823724+0000 mon.a (mon.0) 152 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: cluster 2024-03-07T02:57:58.873208+0000 mon.a (mon.0) 153 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: cluster 2024-03-07T02:57:58.881260+0000 mon.a (mon.0) 154 : cluster 0 monmap e2: 2 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T02:57:59.228 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: cluster 2024-03-07T02:57:58.898931+0000 mon.a (mon.0) 155 : cluster 0 fsmap 2024-03-07T02:57:59.229 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: cluster 2024-03-07T02:57:58.899015+0000 mon.a (mon.0) 156 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T02:57:59.229 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: cluster 2024-03-07T02:57:58.899301+0000 mon.a (mon.0) 157 : cluster 0 mgrmap e13: y(active, since 71s) 2024-03-07T02:57:59.229 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: cluster 2024-03-07T02:57:58.906285+0000 mon.a (mon.0) 158 : cluster 1 overall HEALTH_OK 2024-03-07T02:57:59.229 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: audit 2024-03-07T02:57:58.911105+0000 mon.a (mon.0) 159 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:59.229 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:57:58 smithi189 bash[23614]: audit 2024-03-07T02:57:58.918481+0000 mon.a (mon.0) 160 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: cluster 2024-03-07T02:57:53.234660+0000 mgr.y (mgr.14152) 49 : cluster 0 pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: audit 2024-03-07T02:57:53.827894+0000 mon.a (mon.0) 143 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T02:57:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: cluster 2024-03-07T02:57:53.828237+0000 mon.a (mon.0) 144 : cluster 1 mon.a calling monitor election 2024-03-07T02:57:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: audit 2024-03-07T02:57:53.851274+0000 mon.a (mon.0) 145 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:57:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: audit 2024-03-07T02:57:54.822757+0000 mon.a (mon.0) 146 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:57:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: cluster 2024-03-07T02:57:55.235096+0000 mgr.y (mgr.14152) 50 : cluster 0 pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: audit 2024-03-07T02:57:55.823112+0000 mon.a (mon.0) 147 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:57:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: cluster 2024-03-07T02:57:55.826124+0000 mon.b (mon.1) 1 : cluster 1 mon.b calling monitor election 2024-03-07T02:57:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: audit 2024-03-07T02:57:56.823702+0000 mon.a (mon.0) 148 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:57:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: cluster 2024-03-07T02:57:57.235517+0000 mgr.y (mgr.14152) 51 : cluster 0 pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:57:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: audit 2024-03-07T02:57:57.688277+0000 mon.a (mon.0) 149 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:57:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: audit 2024-03-07T02:57:57.823668+0000 mon.a (mon.0) 150 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:57:59.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: audit 2024-03-07T02:57:58.689153+0000 mon.a (mon.0) 151 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:57:59.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: audit 2024-03-07T02:57:58.823724+0000 mon.a (mon.0) 152 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:57:59.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: cluster 2024-03-07T02:57:58.873208+0000 mon.a (mon.0) 153 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-07T02:57:59.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: cluster 2024-03-07T02:57:58.881260+0000 mon.a (mon.0) 154 : cluster 0 monmap e2: 2 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T02:57:59.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: cluster 2024-03-07T02:57:58.898931+0000 mon.a (mon.0) 155 : cluster 0 fsmap 2024-03-07T02:57:59.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: cluster 2024-03-07T02:57:58.899015+0000 mon.a (mon.0) 156 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T02:57:59.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: cluster 2024-03-07T02:57:58.899301+0000 mon.a (mon.0) 157 : cluster 0 mgrmap e13: y(active, since 71s) 2024-03-07T02:57:59.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: cluster 2024-03-07T02:57:58.906285+0000 mon.a (mon.0) 158 : cluster 1 overall HEALTH_OK 2024-03-07T02:57:59.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: audit 2024-03-07T02:57:58.911105+0000 mon.a (mon.0) 159 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:57:59.323 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:57:58 smithi145 bash[21339]: audit 2024-03-07T02:57:58.918481+0000 mon.a (mon.0) 160 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:00.321 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:57:59 smithi145 bash[21623]: debug 2024-03-07T02:57:59.820+0000 7fc992647700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-03-07T02:58:05.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:57:59.235913+0000 mgr.y (mgr.14152) 52 : cluster 0 pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:05.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: audit 2024-03-07T02:57:59.696171+0000 mon.a (mon.0) 165 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T02:58:05.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:57:59.696879+0000 mon.a (mon.0) 166 : cluster 1 mon.a calling monitor election 2024-03-07T02:58:05.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:57:59.698224+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-03-07T02:58:05.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: audit 2024-03-07T02:57:59.715664+0000 mon.a (mon.0) 167 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:58:05.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: audit 2024-03-07T02:57:59.715952+0000 mon.a (mon.0) 168 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:05.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: audit 2024-03-07T02:58:00.689759+0000 mon.a (mon.0) 169 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:05.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:58:01.236358+0000 mgr.y (mgr.14152) 53 : cluster 0 pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:05.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: audit 2024-03-07T02:58:01.689967+0000 mon.a (mon.0) 170 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:05.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: audit 2024-03-07T02:58:02.689934+0000 mon.a (mon.0) 171 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:05.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:58:03.236770+0000 mgr.y (mgr.14152) 54 : cluster 0 pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:05.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: audit 2024-03-07T02:58:03.690142+0000 mon.a (mon.0) 172 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:05.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: audit 2024-03-07T02:58:04.690228+0000 mon.a (mon.0) 173 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:05.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:58:04.715577+0000 mon.a (mon.0) 174 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-07T02:58:05.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:58:04.721570+0000 mon.a (mon.0) 175 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T02:58:05.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:58:04.722719+0000 mon.a (mon.0) 176 : cluster 0 fsmap 2024-03-07T02:58:05.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:58:04.722803+0000 mon.a (mon.0) 177 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T02:58:05.074 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:58:04.723130+0000 mon.a (mon.0) 178 : cluster 0 mgrmap e13: y(active, since 77s) 2024-03-07T02:58:05.074 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:58:04.723538+0000 mon.a (mon.0) 179 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-03-07T02:58:05.074 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:58:04.741503+0000 mon.a (mon.0) 180 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-03-07T02:58:05.074 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:58:04.741558+0000 mon.a (mon.0) 181 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-03-07T02:58:05.074 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: cluster 2024-03-07T02:58:04.741612+0000 mon.a (mon.0) 182 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] is down (out of quorum) 2024-03-07T02:58:05.074 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: audit 2024-03-07T02:58:04.746397+0000 mon.a (mon.0) 183 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:05.074 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: audit 2024-03-07T02:58:04.751617+0000 mon.a (mon.0) 184 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:05.074 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: audit 2024-03-07T02:58:04.757096+0000 mon.a (mon.0) 185 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:05.074 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: audit 2024-03-07T02:58:04.763052+0000 mon.a (mon.0) 186 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:05.074 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:04 smithi145 bash[21339]: audit 2024-03-07T02:58:04.768953+0000 mon.a (mon.0) 187 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:05.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:57:59.235913+0000 mgr.y (mgr.14152) 52 : cluster 0 pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:05.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: audit 2024-03-07T02:57:59.696171+0000 mon.a (mon.0) 165 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T02:58:05.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:57:59.696879+0000 mon.a (mon.0) 166 : cluster 1 mon.a calling monitor election 2024-03-07T02:58:05.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:57:59.698224+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-03-07T02:58:05.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: audit 2024-03-07T02:57:59.715664+0000 mon.a (mon.0) 167 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:58:05.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: audit 2024-03-07T02:57:59.715952+0000 mon.a (mon.0) 168 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:05.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: audit 2024-03-07T02:58:00.689759+0000 mon.a (mon.0) 169 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:05.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:58:01.236358+0000 mgr.y (mgr.14152) 53 : cluster 0 pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:05.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: audit 2024-03-07T02:58:01.689967+0000 mon.a (mon.0) 170 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:05.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: audit 2024-03-07T02:58:02.689934+0000 mon.a (mon.0) 171 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:05.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:58:03.236770+0000 mgr.y (mgr.14152) 54 : cluster 0 pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:05.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: audit 2024-03-07T02:58:03.690142+0000 mon.a (mon.0) 172 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:05.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: audit 2024-03-07T02:58:04.690228+0000 mon.a (mon.0) 173 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:05.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:58:04.715577+0000 mon.a (mon.0) 174 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-07T02:58:05.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:58:04.721570+0000 mon.a (mon.0) 175 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T02:58:05.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:58:04.722719+0000 mon.a (mon.0) 176 : cluster 0 fsmap 2024-03-07T02:58:05.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:58:04.722803+0000 mon.a (mon.0) 177 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T02:58:05.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:58:04.723130+0000 mon.a (mon.0) 178 : cluster 0 mgrmap e13: y(active, since 77s) 2024-03-07T02:58:05.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:58:04.723538+0000 mon.a (mon.0) 179 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-03-07T02:58:05.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:58:04.741503+0000 mon.a (mon.0) 180 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-03-07T02:58:05.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:58:04.741558+0000 mon.a (mon.0) 181 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-03-07T02:58:05.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: cluster 2024-03-07T02:58:04.741612+0000 mon.a (mon.0) 182 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] is down (out of quorum) 2024-03-07T02:58:05.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: audit 2024-03-07T02:58:04.746397+0000 mon.a (mon.0) 183 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:05.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: audit 2024-03-07T02:58:04.751617+0000 mon.a (mon.0) 184 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:05.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: audit 2024-03-07T02:58:04.757096+0000 mon.a (mon.0) 185 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:05.093 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: audit 2024-03-07T02:58:04.763052+0000 mon.a (mon.0) 186 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:05.093 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:04 smithi189 bash[23614]: audit 2024-03-07T02:58:04.768953+0000 mon.a (mon.0) 187 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:05.330 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-03-07T02:58:05.330 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":3,"fsid":"8947a0f8-dc2d-11ee-95c6-87774f69a715","modified":"2024-03-07T02:57:59.689965Z","created":"2024-03-07T02:55:37.551545Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:3300","nonce":0},{"type":"v1","addr":"172.21.15.145:6789","nonce":0}]},"addr":"172.21.15.145:6789/0","public_addr":"172.21.15.145:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:3300","nonce":0},{"type":"v1","addr":"172.21.15.189:6789","nonce":0}]},"addr":"172.21.15.189:6789/0","public_addr":"172.21.15.189:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:3301","nonce":0},{"type":"v1","addr":"172.21.15.145:6790","nonce":0}]},"addr":"172.21.15.145:6790/0","public_addr":"172.21.15.145:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-03-07T02:58:05.333 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 3 2024-03-07T02:58:05.935 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-03-07T02:58:05.935 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph config generate-minimal-conf 2024-03-07T02:58:06.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:05 smithi145 bash[21339]: audit 2024-03-07T02:58:05.329957+0000 mon.a (mon.0) 188 : audit 0 from='client.? 172.21.15.189:0/95996004' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-07T02:58:06.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:05 smithi145 bash[21339]: audit 2024-03-07T02:58:05.351439+0000 mon.a (mon.0) 189 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:06.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:05 smithi145 bash[21339]: audit 2024-03-07T02:58:05.353529+0000 mon.a (mon.0) 190 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:06.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:05 smithi145 bash[21339]: audit 2024-03-07T02:58:05.690232+0000 mon.a (mon.0) 191 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:06.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:05 smithi145 bash[21339]: audit 2024-03-07T02:58:05.742961+0000 mon.a (mon.0) 192 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:06.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:05 smithi145 bash[21339]: audit 2024-03-07T02:58:05.747912+0000 mon.a (mon.0) 193 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:06.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:05 smithi145 bash[21339]: audit 2024-03-07T02:58:05.753110+0000 mon.a (mon.0) 194 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:06.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:05 smithi145 bash[21339]: audit 2024-03-07T02:58:05.758369+0000 mon.a (mon.0) 195 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:06.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:05 smithi145 bash[21339]: audit 2024-03-07T02:58:05.764367+0000 mon.a (mon.0) 196 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:06.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:05 smithi189 bash[23614]: audit 2024-03-07T02:58:05.329957+0000 mon.a (mon.0) 188 : audit 0 from='client.? 172.21.15.189:0/95996004' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-07T02:58:06.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:05 smithi189 bash[23614]: audit 2024-03-07T02:58:05.351439+0000 mon.a (mon.0) 189 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:06.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:05 smithi189 bash[23614]: audit 2024-03-07T02:58:05.353529+0000 mon.a (mon.0) 190 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:06.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:05 smithi189 bash[23614]: audit 2024-03-07T02:58:05.690232+0000 mon.a (mon.0) 191 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:06.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:05 smithi189 bash[23614]: audit 2024-03-07T02:58:05.742961+0000 mon.a (mon.0) 192 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:06.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:05 smithi189 bash[23614]: audit 2024-03-07T02:58:05.747912+0000 mon.a (mon.0) 193 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:06.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:05 smithi189 bash[23614]: audit 2024-03-07T02:58:05.753110+0000 mon.a (mon.0) 194 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:06.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:05 smithi189 bash[23614]: audit 2024-03-07T02:58:05.758369+0000 mon.a (mon.0) 195 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:06.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:05 smithi189 bash[23614]: audit 2024-03-07T02:58:05.764367+0000 mon.a (mon.0) 196 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:07.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[21339]: cluster 2024-03-07T02:58:01.692139+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-03-07T02:58:07.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[21339]: cluster 2024-03-07T02:58:06.722659+0000 mon.c (mon.2) 2 : cluster 1 mon.c calling monitor election 2024-03-07T02:58:07.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[21339]: cluster 2024-03-07T02:58:06.723910+0000 mon.a (mon.0) 205 : cluster 1 mon.a calling monitor election 2024-03-07T02:58:07.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[21339]: cluster 2024-03-07T02:58:06.725087+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2024-03-07T02:58:07.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[21339]: cluster 2024-03-07T02:58:06.749309+0000 mon.a (mon.0) 206 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T02:58:07.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[21339]: cluster 2024-03-07T02:58:06.755464+0000 mon.a (mon.0) 207 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T02:58:07.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[21339]: cluster 2024-03-07T02:58:06.756459+0000 mon.a (mon.0) 208 : cluster 0 fsmap 2024-03-07T02:58:07.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[21339]: cluster 2024-03-07T02:58:06.756529+0000 mon.a (mon.0) 209 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T02:58:07.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[21339]: cluster 2024-03-07T02:58:06.756822+0000 mon.a (mon.0) 210 : cluster 0 mgrmap e13: y(active, since 79s) 2024-03-07T02:58:07.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[21339]: cluster 2024-03-07T02:58:06.757116+0000 mon.a (mon.0) 211 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-07T02:58:07.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[21339]: cluster 2024-03-07T02:58:06.757158+0000 mon.a (mon.0) 212 : cluster 1 Cluster is now healthy 2024-03-07T02:58:07.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[21339]: cluster 2024-03-07T02:58:06.765108+0000 mon.a (mon.0) 213 : cluster 1 overall HEALTH_OK 2024-03-07T02:58:07.073 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:53.234660+0000 mgr.y (mgr.14152) 49 : cluster 0 pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:07.073 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:53.827894+0000 mon.a (mon.0) 143 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T02:58:07.073 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:53.828237+0000 mon.a (mon.0) 144 : cluster 1 mon.a calling monitor election 2024-03-07T02:58:07.073 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:53.851274+0000 mon.a (mon.0) 145 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:54.822757+0000 mon.a (mon.0) 146 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:55.235096+0000 mgr.y (mgr.14152) 50 : cluster 0 pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:55.823112+0000 mon.a (mon.0) 147 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:55.826124+0000 mon.b (mon.1) 1 : cluster 1 mon.b calling monitor election 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:56.823702+0000 mon.a (mon.0) 148 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:57.235517+0000 mgr.y (mgr.14152) 51 : cluster 0 pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:57.688277+0000 mon.a (mon.0) 149 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:57.823668+0000 mon.a (mon.0) 150 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:58.689153+0000 mon.a (mon.0) 151 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:58.823724+0000 mon.a (mon.0) 152 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:58.873208+0000 mon.a (mon.0) 153 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:58.881260+0000 mon.a (mon.0) 154 : cluster 0 monmap e2: 2 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:58.898931+0000 mon.a (mon.0) 155 : cluster 0 fsmap 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:58.899015+0000 mon.a (mon.0) 156 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:58.899301+0000 mon.a (mon.0) 157 : cluster 0 mgrmap e13: y(active, since 71s) 2024-03-07T02:58:07.074 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:58.906285+0000 mon.a (mon.0) 158 : cluster 1 overall HEALTH_OK 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:58.911105+0000 mon.a (mon.0) 159 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:58.918481+0000 mon.a (mon.0) 160 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:59.235913+0000 mgr.y (mgr.14152) 52 : cluster 0 pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:59.696171+0000 mon.a (mon.0) 165 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:59.696879+0000 mon.a (mon.0) 166 : cluster 1 mon.a calling monitor election 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:57:59.698224+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:59.715664+0000 mon.a (mon.0) 167 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:57:59.715952+0000 mon.a (mon.0) 168 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:00.689759+0000 mon.a (mon.0) 169 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:01.236358+0000 mgr.y (mgr.14152) 53 : cluster 0 pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:01.689967+0000 mon.a (mon.0) 170 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:02.689934+0000 mon.a (mon.0) 171 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:03.236770+0000 mgr.y (mgr.14152) 54 : cluster 0 pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:03.690142+0000 mon.a (mon.0) 172 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:04.690228+0000 mon.a (mon.0) 173 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:07.075 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:04.715577+0000 mon.a (mon.0) 174 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:04.721570+0000 mon.a (mon.0) 175 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:04.722719+0000 mon.a (mon.0) 176 : cluster 0 fsmap 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:04.722803+0000 mon.a (mon.0) 177 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:04.723130+0000 mon.a (mon.0) 178 : cluster 0 mgrmap e13: y(active, since 77s) 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:04.723538+0000 mon.a (mon.0) 179 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:04.741503+0000 mon.a (mon.0) 180 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:04.741558+0000 mon.a (mon.0) 181 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:04.741612+0000 mon.a (mon.0) 182 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] is down (out of quorum) 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:04.746397+0000 mon.a (mon.0) 183 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:04.751617+0000 mon.a (mon.0) 184 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:04.757096+0000 mon.a (mon.0) 185 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:04.763052+0000 mon.a (mon.0) 186 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:04.768953+0000 mon.a (mon.0) 187 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:05.329957+0000 mon.a (mon.0) 188 : audit 0 from='client.? 172.21.15.189:0/95996004' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-07T02:58:07.076 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:05.351439+0000 mon.a (mon.0) 189 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:05.353529+0000 mon.a (mon.0) 190 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:05.690232+0000 mon.a (mon.0) 191 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:05.742961+0000 mon.a (mon.0) 192 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:05.747912+0000 mon.a (mon.0) 193 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:05.753110+0000 mon.a (mon.0) 194 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:05.758369+0000 mon.a (mon.0) 195 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: audit 2024-03-07T02:58:05.764367+0000 mon.a (mon.0) 196 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:01.692139+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:06.722659+0000 mon.c (mon.2) 2 : cluster 1 mon.c calling monitor election 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:06.723910+0000 mon.a (mon.0) 205 : cluster 1 mon.a calling monitor election 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:06.725087+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:06.749309+0000 mon.a (mon.0) 206 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:06.755464+0000 mon.a (mon.0) 207 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:06.756459+0000 mon.a (mon.0) 208 : cluster 0 fsmap 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:06.756529+0000 mon.a (mon.0) 209 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:06.756822+0000 mon.a (mon.0) 210 : cluster 0 mgrmap e13: y(active, since 79s) 2024-03-07T02:58:07.077 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:06.757116+0000 mon.a (mon.0) 211 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-07T02:58:07.078 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:06.757158+0000 mon.a (mon.0) 212 : cluster 1 Cluster is now healthy 2024-03-07T02:58:07.078 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:06 smithi145 bash[30906]: cluster 2024-03-07T02:58:06.765108+0000 mon.a (mon.0) 213 : cluster 1 overall HEALTH_OK 2024-03-07T02:58:07.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:06 smithi189 bash[23614]: cluster 2024-03-07T02:58:01.692139+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-03-07T02:58:07.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:06 smithi189 bash[23614]: cluster 2024-03-07T02:58:06.722659+0000 mon.c (mon.2) 2 : cluster 1 mon.c calling monitor election 2024-03-07T02:58:07.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:06 smithi189 bash[23614]: cluster 2024-03-07T02:58:06.723910+0000 mon.a (mon.0) 205 : cluster 1 mon.a calling monitor election 2024-03-07T02:58:07.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:06 smithi189 bash[23614]: cluster 2024-03-07T02:58:06.725087+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2024-03-07T02:58:07.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:06 smithi189 bash[23614]: cluster 2024-03-07T02:58:06.749309+0000 mon.a (mon.0) 206 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T02:58:07.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:06 smithi189 bash[23614]: cluster 2024-03-07T02:58:06.755464+0000 mon.a (mon.0) 207 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T02:58:07.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:06 smithi189 bash[23614]: cluster 2024-03-07T02:58:06.756459+0000 mon.a (mon.0) 208 : cluster 0 fsmap 2024-03-07T02:58:07.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:06 smithi189 bash[23614]: cluster 2024-03-07T02:58:06.756529+0000 mon.a (mon.0) 209 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T02:58:07.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:06 smithi189 bash[23614]: cluster 2024-03-07T02:58:06.756822+0000 mon.a (mon.0) 210 : cluster 0 mgrmap e13: y(active, since 79s) 2024-03-07T02:58:07.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:06 smithi189 bash[23614]: cluster 2024-03-07T02:58:06.757116+0000 mon.a (mon.0) 211 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-07T02:58:07.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:06 smithi189 bash[23614]: cluster 2024-03-07T02:58:06.757158+0000 mon.a (mon.0) 212 : cluster 1 Cluster is now healthy 2024-03-07T02:58:07.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:06 smithi189 bash[23614]: cluster 2024-03-07T02:58:06.765108+0000 mon.a (mon.0) 213 : cluster 1 overall HEALTH_OK 2024-03-07T02:58:08.926 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[21339]: cluster 2024-03-07T02:58:07.237484+0000 mgr.y (mgr.14152) 62 : cluster 0 pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:08.926 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[21339]: audit 2024-03-07T02:58:07.690853+0000 mon.a (mon.0) 214 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:08.926 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[21339]: audit 2024-03-07T02:58:07.838819+0000 mon.a (mon.0) 215 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:08.926 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[21339]: audit 2024-03-07T02:58:07.844441+0000 mon.a (mon.0) 216 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:08.927 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[21339]: cephadm 2024-03-07T02:58:07.877789+0000 mgr.y (mgr.14152) 63 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-03-07T02:58:08.927 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[21339]: audit 2024-03-07T02:58:07.878348+0000 mon.a (mon.0) 217 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T02:58:08.927 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[21339]: audit 2024-03-07T02:58:07.879963+0000 mon.a (mon.0) 218 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T02:58:08.927 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[21339]: audit 2024-03-07T02:58:07.881436+0000 mon.a (mon.0) 219 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:08.927 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[21339]: cephadm 2024-03-07T02:58:07.883258+0000 mgr.y (mgr.14152) 64 : cephadm 1 Reconfiguring daemon mon.c on smithi145 2024-03-07T02:58:08.927 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[30906]: cluster 2024-03-07T02:58:07.237484+0000 mgr.y (mgr.14152) 62 : cluster 0 pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:08.927 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[30906]: audit 2024-03-07T02:58:07.690853+0000 mon.a (mon.0) 214 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:08.927 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[30906]: audit 2024-03-07T02:58:07.838819+0000 mon.a (mon.0) 215 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:08.927 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[30906]: audit 2024-03-07T02:58:07.844441+0000 mon.a (mon.0) 216 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:08.927 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[30906]: cephadm 2024-03-07T02:58:07.877789+0000 mgr.y (mgr.14152) 63 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-03-07T02:58:08.927 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[30906]: audit 2024-03-07T02:58:07.878348+0000 mon.a (mon.0) 217 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T02:58:08.928 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[30906]: audit 2024-03-07T02:58:07.879963+0000 mon.a (mon.0) 218 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T02:58:08.928 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[30906]: audit 2024-03-07T02:58:07.881436+0000 mon.a (mon.0) 219 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:08.928 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:08 smithi145 bash[30906]: cephadm 2024-03-07T02:58:07.883258+0000 mgr.y (mgr.14152) 64 : cephadm 1 Reconfiguring daemon mon.c on smithi145 2024-03-07T02:58:09.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:08 smithi189 bash[23614]: cluster 2024-03-07T02:58:07.237484+0000 mgr.y (mgr.14152) 62 : cluster 0 pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:09.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:08 smithi189 bash[23614]: audit 2024-03-07T02:58:07.690853+0000 mon.a (mon.0) 214 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T02:58:09.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:08 smithi189 bash[23614]: audit 2024-03-07T02:58:07.838819+0000 mon.a (mon.0) 215 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:09.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:08 smithi189 bash[23614]: audit 2024-03-07T02:58:07.844441+0000 mon.a (mon.0) 216 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:09.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:08 smithi189 bash[23614]: cephadm 2024-03-07T02:58:07.877789+0000 mgr.y (mgr.14152) 63 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-03-07T02:58:09.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:08 smithi189 bash[23614]: audit 2024-03-07T02:58:07.878348+0000 mon.a (mon.0) 217 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T02:58:09.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:08 smithi189 bash[23614]: audit 2024-03-07T02:58:07.879963+0000 mon.a (mon.0) 218 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T02:58:09.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:08 smithi189 bash[23614]: audit 2024-03-07T02:58:07.881436+0000 mon.a (mon.0) 219 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:09.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:08 smithi189 bash[23614]: cephadm 2024-03-07T02:58:07.883258+0000 mgr.y (mgr.14152) 64 : cephadm 1 Reconfiguring daemon mon.c on smithi145 2024-03-07T02:58:10.607 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T02:58:11.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[21339]: cluster 2024-03-07T02:58:09.237907+0000 mgr.y (mgr.14152) 65 : cluster 0 pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:11.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[21339]: audit 2024-03-07T02:58:09.745408+0000 mon.a (mon.0) 220 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:11.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[21339]: audit 2024-03-07T02:58:09.752008+0000 mon.a (mon.0) 221 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:11.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[21339]: cephadm 2024-03-07T02:58:09.802909+0000 mgr.y (mgr.14152) 66 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-03-07T02:58:11.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[21339]: audit 2024-03-07T02:58:09.803385+0000 mon.a (mon.0) 222 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T02:58:11.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[21339]: audit 2024-03-07T02:58:09.804949+0000 mon.a (mon.0) 223 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T02:58:11.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[21339]: audit 2024-03-07T02:58:09.806238+0000 mon.a (mon.0) 224 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:11.072 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[21339]: cephadm 2024-03-07T02:58:09.807651+0000 mgr.y (mgr.14152) 67 : cephadm 1 Reconfiguring daemon mon.b on smithi189 2024-03-07T02:58:11.073 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[30906]: cluster 2024-03-07T02:58:09.237907+0000 mgr.y (mgr.14152) 65 : cluster 0 pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:11.073 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[30906]: audit 2024-03-07T02:58:09.745408+0000 mon.a (mon.0) 220 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:11.073 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[30906]: audit 2024-03-07T02:58:09.752008+0000 mon.a (mon.0) 221 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:11.073 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[30906]: cephadm 2024-03-07T02:58:09.802909+0000 mgr.y (mgr.14152) 66 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-03-07T02:58:11.073 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[30906]: audit 2024-03-07T02:58:09.803385+0000 mon.a (mon.0) 222 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T02:58:11.073 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[30906]: audit 2024-03-07T02:58:09.804949+0000 mon.a (mon.0) 223 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T02:58:11.073 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[30906]: audit 2024-03-07T02:58:09.806238+0000 mon.a (mon.0) 224 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:11.073 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:10 smithi145 bash[30906]: cephadm 2024-03-07T02:58:09.807651+0000 mgr.y (mgr.14152) 67 : cephadm 1 Reconfiguring daemon mon.b on smithi189 2024-03-07T02:58:11.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:10 smithi189 bash[23614]: cluster 2024-03-07T02:58:09.237907+0000 mgr.y (mgr.14152) 65 : cluster 0 pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:11.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:10 smithi189 bash[23614]: audit 2024-03-07T02:58:09.745408+0000 mon.a (mon.0) 220 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:11.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:10 smithi189 bash[23614]: audit 2024-03-07T02:58:09.752008+0000 mon.a (mon.0) 221 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:11.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:10 smithi189 bash[23614]: cephadm 2024-03-07T02:58:09.802909+0000 mgr.y (mgr.14152) 66 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-03-07T02:58:11.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:10 smithi189 bash[23614]: audit 2024-03-07T02:58:09.803385+0000 mon.a (mon.0) 222 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T02:58:11.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:10 smithi189 bash[23614]: audit 2024-03-07T02:58:09.804949+0000 mon.a (mon.0) 223 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T02:58:11.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:10 smithi189 bash[23614]: audit 2024-03-07T02:58:09.806238+0000 mon.a (mon.0) 224 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:11.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:10 smithi189 bash[23614]: cephadm 2024-03-07T02:58:09.807651+0000 mgr.y (mgr.14152) 67 : cephadm 1 Reconfiguring daemon mon.b on smithi189 2024-03-07T02:58:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:12 smithi189 bash[23614]: cluster 2024-03-07T02:58:11.238407+0000 mgr.y (mgr.14152) 68 : cluster 0 pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:12 smithi189 bash[23614]: audit 2024-03-07T02:58:11.326115+0000 mon.a (mon.0) 225 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:12 smithi189 bash[23614]: audit 2024-03-07T02:58:11.330719+0000 mon.a (mon.0) 226 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:12 smithi189 bash[23614]: audit 2024-03-07T02:58:11.376284+0000 mon.a (mon.0) 227 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:58:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:12 smithi189 bash[23614]: audit 2024-03-07T02:58:11.377925+0000 mon.a (mon.0) 228 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:12 smithi189 bash[23614]: audit 2024-03-07T02:58:11.378691+0000 mon.a (mon.0) 229 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:12 smithi189 bash[23614]: audit 2024-03-07T02:58:11.384760+0000 mon.a (mon.0) 230 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:12.821 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[30906]: cluster 2024-03-07T02:58:11.238407+0000 mgr.y (mgr.14152) 68 : cluster 0 pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:12.822 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[30906]: audit 2024-03-07T02:58:11.326115+0000 mon.a (mon.0) 225 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:12.822 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[30906]: audit 2024-03-07T02:58:11.330719+0000 mon.a (mon.0) 226 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:12.822 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[30906]: audit 2024-03-07T02:58:11.376284+0000 mon.a (mon.0) 227 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:58:12.822 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[30906]: audit 2024-03-07T02:58:11.377925+0000 mon.a (mon.0) 228 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:12.822 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[30906]: audit 2024-03-07T02:58:11.378691+0000 mon.a (mon.0) 229 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:12.822 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[30906]: audit 2024-03-07T02:58:11.384760+0000 mon.a (mon.0) 230 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:12.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[21339]: cluster 2024-03-07T02:58:11.238407+0000 mgr.y (mgr.14152) 68 : cluster 0 pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:12.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[21339]: audit 2024-03-07T02:58:11.326115+0000 mon.a (mon.0) 225 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:12.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[21339]: audit 2024-03-07T02:58:11.330719+0000 mon.a (mon.0) 226 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:12.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[21339]: audit 2024-03-07T02:58:11.376284+0000 mon.a (mon.0) 227 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:58:12.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[21339]: audit 2024-03-07T02:58:11.377925+0000 mon.a (mon.0) 228 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:12.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[21339]: audit 2024-03-07T02:58:11.378691+0000 mon.a (mon.0) 229 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:12.823 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:12 smithi145 bash[21339]: audit 2024-03-07T02:58:11.384760+0000 mon.a (mon.0) 230 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:13.268 INFO:teuthology.orchestra.run.smithi145.stdout:# minimal ceph.conf for 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:58:13.268 INFO:teuthology.orchestra.run.smithi145.stdout:[global] 2024-03-07T02:58:13.268 INFO:teuthology.orchestra.run.smithi145.stdout: fsid = 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:58:13.269 INFO:teuthology.orchestra.run.smithi145.stdout: mon_host = [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] 2024-03-07T02:58:13.952 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-03-07T02:58:13.952 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:58:13.952 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/ceph/ceph.conf 2024-03-07T02:58:13.966 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:58:13.966 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-07T02:58:14.025 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-07T02:58:14.025 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/ceph/ceph.conf 2024-03-07T02:58:14.040 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-07T02:58:14.040 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-07T02:58:14.096 INFO:tasks.cephadm:Adding mgr.y on smithi145 2024-03-07T02:58:14.096 INFO:tasks.cephadm:Adding mgr.x on smithi189 2024-03-07T02:58:14.096 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch apply mgr '2;smithi145=y;smithi189=x' 2024-03-07T02:58:14.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:14 smithi145 bash[21339]: audit 2024-03-07T02:58:13.268333+0000 mon.a (mon.0) 231 : audit 0 from='client.? 172.21.15.145:0/3142217168' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:14.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:14 smithi145 bash[30906]: audit 2024-03-07T02:58:13.268333+0000 mon.a (mon.0) 231 : audit 0 from='client.? 172.21.15.145:0/3142217168' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:14.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:14 smithi189 bash[23614]: audit 2024-03-07T02:58:13.268333+0000 mon.a (mon.0) 231 : audit 0 from='client.? 172.21.15.145:0/3142217168' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:15.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:15 smithi145 bash[21339]: cluster 2024-03-07T02:58:13.238789+0000 mgr.y (mgr.14152) 69 : cluster 0 pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:15.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:15 smithi145 bash[30906]: cluster 2024-03-07T02:58:13.238789+0000 mgr.y (mgr.14152) 69 : cluster 0 pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:15.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:15 smithi189 bash[23614]: cluster 2024-03-07T02:58:13.238789+0000 mgr.y (mgr.14152) 69 : cluster 0 pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:17.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:17 smithi145 bash[21339]: cluster 2024-03-07T02:58:15.239194+0000 mgr.y (mgr.14152) 70 : cluster 0 pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:17.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:17 smithi145 bash[30906]: cluster 2024-03-07T02:58:15.239194+0000 mgr.y (mgr.14152) 70 : cluster 0 pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:17.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:17 smithi189 bash[23614]: cluster 2024-03-07T02:58:15.239194+0000 mgr.y (mgr.14152) 70 : cluster 0 pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:17.795 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T02:58:19.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:19 smithi145 bash[21339]: cluster 2024-03-07T02:58:17.239584+0000 mgr.y (mgr.14152) 71 : cluster 0 pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:19.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:19 smithi145 bash[30906]: cluster 2024-03-07T02:58:17.239584+0000 mgr.y (mgr.14152) 71 : cluster 0 pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:19.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:19 smithi189 bash[23614]: cluster 2024-03-07T02:58:17.239584+0000 mgr.y (mgr.14152) 71 : cluster 0 pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:20.142 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled mgr update... 2024-03-07T02:58:20.920 DEBUG:teuthology.orchestra.run.smithi189:mgr.x> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.x.service 2024-03-07T02:58:20.923 INFO:tasks.cephadm:Deploying OSDs... 2024-03-07T02:58:20.923 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T02:58:20.924 DEBUG:teuthology.orchestra.run.smithi145:> dd if=/scratch_devs of=/dev/stdout 2024-03-07T02:58:20.930 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-07T02:58:20.930 DEBUG:teuthology.orchestra.run.smithi145:> stat /dev/vg_nvme/lv_1 2024-03-07T02:58:20.978 INFO:teuthology.orchestra.run.smithi145.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-07T02:58:20.978 INFO:teuthology.orchestra.run.smithi145.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T02:58:20.978 INFO:teuthology.orchestra.run.smithi145.stdout:Device: 5h/5d Inode: 890 Links: 1 2024-03-07T02:58:20.978 INFO:teuthology.orchestra.run.smithi145.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T02:58:20.978 INFO:teuthology.orchestra.run.smithi145.stdout:Access: 2024-03-07 02:45:51.681670905 +0000 2024-03-07T02:58:20.978 INFO:teuthology.orchestra.run.smithi145.stdout:Modify: 2024-03-07 02:45:51.677671592 +0000 2024-03-07T02:58:20.978 INFO:teuthology.orchestra.run.smithi145.stdout:Change: 2024-03-07 02:45:51.677671592 +0000 2024-03-07T02:58:20.978 INFO:teuthology.orchestra.run.smithi145.stdout: Birth: - 2024-03-07T02:58:20.979 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-07T02:58:21.037 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records in 2024-03-07T02:58:21.037 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records out 2024-03-07T02:58:21.037 INFO:teuthology.orchestra.run.smithi145.stderr:512 bytes copied, 0.000443645 s, 1.2 MB/s 2024-03-07T02:58:21.038 DEBUG:teuthology.orchestra.run.smithi145:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-07T02:58:21.087 DEBUG:teuthology.orchestra.run.smithi145:> stat /dev/vg_nvme/lv_2 2024-03-07T02:58:21.134 INFO:teuthology.orchestra.run.smithi145.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-07T02:58:21.134 INFO:teuthology.orchestra.run.smithi145.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T02:58:21.134 INFO:teuthology.orchestra.run.smithi145.stdout:Device: 5h/5d Inode: 902 Links: 1 2024-03-07T02:58:21.134 INFO:teuthology.orchestra.run.smithi145.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T02:58:21.134 INFO:teuthology.orchestra.run.smithi145.stdout:Access: 2024-03-07 02:45:52.425545104 +0000 2024-03-07T02:58:21.134 INFO:teuthology.orchestra.run.smithi145.stdout:Modify: 2024-03-07 02:45:52.161589175 +0000 2024-03-07T02:58:21.134 INFO:teuthology.orchestra.run.smithi145.stdout:Change: 2024-03-07 02:45:52.161589175 +0000 2024-03-07T02:58:21.134 INFO:teuthology.orchestra.run.smithi145.stdout: Birth: - 2024-03-07T02:58:21.134 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-07T02:58:21.192 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records in 2024-03-07T02:58:21.192 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records out 2024-03-07T02:58:21.192 INFO:teuthology.orchestra.run.smithi145.stderr:512 bytes copied, 0.000380828 s, 1.3 MB/s 2024-03-07T02:58:21.192 DEBUG:teuthology.orchestra.run.smithi145:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-07T02:58:21.240 DEBUG:teuthology.orchestra.run.smithi145:> stat /dev/vg_nvme/lv_3 2024-03-07T02:58:21.286 INFO:teuthology.orchestra.run.smithi145.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-07T02:58:21.286 INFO:teuthology.orchestra.run.smithi145.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T02:58:21.286 INFO:teuthology.orchestra.run.smithi145.stdout:Device: 5h/5d Inode: 924 Links: 1 2024-03-07T02:58:21.286 INFO:teuthology.orchestra.run.smithi145.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T02:58:21.286 INFO:teuthology.orchestra.run.smithi145.stdout:Access: 2024-03-07 02:45:52.909464309 +0000 2024-03-07T02:58:21.286 INFO:teuthology.orchestra.run.smithi145.stdout:Modify: 2024-03-07 02:45:52.649507712 +0000 2024-03-07T02:58:21.286 INFO:teuthology.orchestra.run.smithi145.stdout:Change: 2024-03-07 02:45:52.649507712 +0000 2024-03-07T02:58:21.286 INFO:teuthology.orchestra.run.smithi145.stdout: Birth: - 2024-03-07T02:58:21.286 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-07T02:58:21.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[21339]: cluster 2024-03-07T02:58:19.240036+0000 mgr.y (mgr.14152) 72 : cluster 0 pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:21.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[21339]: audit 2024-03-07T02:58:20.131052+0000 mgr.y (mgr.14152) 73 : audit 0 from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi145=y;smithi189=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:58:21.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[21339]: cephadm 2024-03-07T02:58:20.134538+0000 mgr.y (mgr.14152) 74 : cephadm 1 Saving service mgr spec with placement smithi145=y;smithi189=x;count:2 2024-03-07T02:58:21.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[21339]: audit 2024-03-07T02:58:20.141031+0000 mon.a (mon.0) 232 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:21.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[21339]: audit 2024-03-07T02:58:20.171129+0000 mon.a (mon.0) 233 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:58:21.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[21339]: audit 2024-03-07T02:58:20.174919+0000 mon.a (mon.0) 234 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:21.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[21339]: audit 2024-03-07T02:58:20.176991+0000 mon.a (mon.0) 235 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:21.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[21339]: audit 2024-03-07T02:58:20.188980+0000 mon.a (mon.0) 236 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:21.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[21339]: audit 2024-03-07T02:58:20.192087+0000 mon.a (mon.0) 237 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T02:58:21.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[21339]: audit 2024-03-07T02:58:20.197847+0000 mon.a (mon.0) 238 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-07T02:58:21.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[21339]: audit 2024-03-07T02:58:20.199417+0000 mon.a (mon.0) 239 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T02:58:21.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[21339]: audit 2024-03-07T02:58:20.201155+0000 mon.a (mon.0) 240 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:21.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[21339]: cephadm 2024-03-07T02:58:20.203092+0000 mgr.y (mgr.14152) 75 : cephadm 1 Deploying daemon mgr.x on smithi189 2024-03-07T02:58:21.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[30906]: cluster 2024-03-07T02:58:19.240036+0000 mgr.y (mgr.14152) 72 : cluster 0 pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:21.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[30906]: audit 2024-03-07T02:58:20.131052+0000 mgr.y (mgr.14152) 73 : audit 0 from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi145=y;smithi189=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:58:21.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[30906]: cephadm 2024-03-07T02:58:20.134538+0000 mgr.y (mgr.14152) 74 : cephadm 1 Saving service mgr spec with placement smithi145=y;smithi189=x;count:2 2024-03-07T02:58:21.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[30906]: audit 2024-03-07T02:58:20.141031+0000 mon.a (mon.0) 232 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:21.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[30906]: audit 2024-03-07T02:58:20.171129+0000 mon.a (mon.0) 233 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:58:21.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[30906]: audit 2024-03-07T02:58:20.174919+0000 mon.a (mon.0) 234 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:21.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[30906]: audit 2024-03-07T02:58:20.176991+0000 mon.a (mon.0) 235 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:21.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[30906]: audit 2024-03-07T02:58:20.188980+0000 mon.a (mon.0) 236 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:21.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[30906]: audit 2024-03-07T02:58:20.192087+0000 mon.a (mon.0) 237 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T02:58:21.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[30906]: audit 2024-03-07T02:58:20.197847+0000 mon.a (mon.0) 238 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-07T02:58:21.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[30906]: audit 2024-03-07T02:58:20.199417+0000 mon.a (mon.0) 239 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T02:58:21.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[30906]: audit 2024-03-07T02:58:20.201155+0000 mon.a (mon.0) 240 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:21.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:21 smithi145 bash[30906]: cephadm 2024-03-07T02:58:20.203092+0000 mgr.y (mgr.14152) 75 : cephadm 1 Deploying daemon mgr.x on smithi189 2024-03-07T02:58:21.334 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records in 2024-03-07T02:58:21.334 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records out 2024-03-07T02:58:21.334 INFO:teuthology.orchestra.run.smithi145.stderr:512 bytes copied, 0.000385104 s, 1.3 MB/s 2024-03-07T02:58:21.335 DEBUG:teuthology.orchestra.run.smithi145:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-07T02:58:21.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:21 smithi189 bash[23614]: cluster 2024-03-07T02:58:19.240036+0000 mgr.y (mgr.14152) 72 : cluster 0 pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:21.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:21 smithi189 bash[23614]: audit 2024-03-07T02:58:20.131052+0000 mgr.y (mgr.14152) 73 : audit 0 from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi145=y;smithi189=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:58:21.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:21 smithi189 bash[23614]: cephadm 2024-03-07T02:58:20.134538+0000 mgr.y (mgr.14152) 74 : cephadm 1 Saving service mgr spec with placement smithi145=y;smithi189=x;count:2 2024-03-07T02:58:21.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:21 smithi189 bash[23614]: audit 2024-03-07T02:58:20.141031+0000 mon.a (mon.0) 232 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:21.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:21 smithi189 bash[23614]: audit 2024-03-07T02:58:20.171129+0000 mon.a (mon.0) 233 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:58:21.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:21 smithi189 bash[23614]: audit 2024-03-07T02:58:20.174919+0000 mon.a (mon.0) 234 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:21.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:21 smithi189 bash[23614]: audit 2024-03-07T02:58:20.176991+0000 mon.a (mon.0) 235 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:21.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:21 smithi189 bash[23614]: audit 2024-03-07T02:58:20.188980+0000 mon.a (mon.0) 236 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:21.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:21 smithi189 bash[23614]: audit 2024-03-07T02:58:20.192087+0000 mon.a (mon.0) 237 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T02:58:21.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:21 smithi189 bash[23614]: audit 2024-03-07T02:58:20.197847+0000 mon.a (mon.0) 238 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-07T02:58:21.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:21 smithi189 bash[23614]: audit 2024-03-07T02:58:20.199417+0000 mon.a (mon.0) 239 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T02:58:21.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:21 smithi189 bash[23614]: audit 2024-03-07T02:58:20.201155+0000 mon.a (mon.0) 240 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:21.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:21 smithi189 bash[23614]: cephadm 2024-03-07T02:58:20.203092+0000 mgr.y (mgr.14152) 75 : cephadm 1 Deploying daemon mgr.x on smithi189 2024-03-07T02:58:21.384 DEBUG:teuthology.orchestra.run.smithi145:> stat /dev/vg_nvme/lv_4 2024-03-07T02:58:21.430 INFO:teuthology.orchestra.run.smithi145.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-07T02:58:21.430 INFO:teuthology.orchestra.run.smithi145.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T02:58:21.430 INFO:teuthology.orchestra.run.smithi145.stdout:Device: 5h/5d Inode: 941 Links: 1 2024-03-07T02:58:21.430 INFO:teuthology.orchestra.run.smithi145.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T02:58:21.430 INFO:teuthology.orchestra.run.smithi145.stdout:Access: 2024-03-07 02:45:53.433378994 +0000 2024-03-07T02:58:21.430 INFO:teuthology.orchestra.run.smithi145.stdout:Modify: 2024-03-07 02:45:53.173421096 +0000 2024-03-07T02:58:21.430 INFO:teuthology.orchestra.run.smithi145.stdout:Change: 2024-03-07 02:45:53.173421096 +0000 2024-03-07T02:58:21.430 INFO:teuthology.orchestra.run.smithi145.stdout: Birth: - 2024-03-07T02:58:21.430 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-07T02:58:21.484 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records in 2024-03-07T02:58:21.484 INFO:teuthology.orchestra.run.smithi145.stderr:1+0 records out 2024-03-07T02:58:21.484 INFO:teuthology.orchestra.run.smithi145.stderr:512 bytes copied, 0.000321713 s, 1.6 MB/s 2024-03-07T02:58:21.486 DEBUG:teuthology.orchestra.run.smithi145:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-07T02:58:21.535 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-07T02:58:21.535 DEBUG:teuthology.orchestra.run.smithi189:> dd if=/scratch_devs of=/dev/stdout 2024-03-07T02:58:21.540 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-07T02:58:21.540 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_1 2024-03-07T02:58:21.587 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-07T02:58:21.587 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T02:58:21.587 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 874 Links: 1 2024-03-07T02:58:21.587 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T02:58:21.587 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-03-07 02:45:50.178942625 +0000 2024-03-07T02:58:21.587 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-03-07 02:45:49.890945198 +0000 2024-03-07T02:58:21.587 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-03-07 02:45:49.890945198 +0000 2024-03-07T02:58:21.587 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: - 2024-03-07T02:58:21.588 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-07T02:58:21.645 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-03-07T02:58:21.645 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-03-07T02:58:21.645 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000446986 s, 1.1 MB/s 2024-03-07T02:58:21.646 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-07T02:58:21.696 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_2 2024-03-07T02:58:21.743 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-07T02:58:21.744 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T02:58:21.744 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 886 Links: 1 2024-03-07T02:58:21.744 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T02:58:21.744 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-03-07 02:45:50.658938338 +0000 2024-03-07T02:58:21.744 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-03-07 02:45:50.414940517 +0000 2024-03-07T02:58:21.744 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-03-07 02:45:50.414940517 +0000 2024-03-07T02:58:21.744 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: - 2024-03-07T02:58:21.744 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-07T02:58:21.801 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-03-07T02:58:21.802 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-03-07T02:58:21.802 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000351756 s, 1.5 MB/s 2024-03-07T02:58:21.803 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-07T02:58:21.852 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_3 2024-03-07T02:58:21.898 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-07T02:58:21.898 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T02:58:21.899 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 911 Links: 1 2024-03-07T02:58:21.899 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T02:58:21.899 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-03-07 02:45:51.122934194 +0000 2024-03-07T02:58:21.899 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-03-07 02:45:50.882936337 +0000 2024-03-07T02:58:21.899 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-03-07 02:45:50.882936337 +0000 2024-03-07T02:58:21.899 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: - 2024-03-07T02:58:21.899 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-07T02:58:21.951 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-03-07T02:58:21.952 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-03-07T02:58:21.952 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000408488 s, 1.3 MB/s 2024-03-07T02:58:21.953 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-07T02:58:22.001 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_4 2024-03-07T02:58:22.047 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-07T02:58:22.047 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T02:58:22.047 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 922 Links: 1 2024-03-07T02:58:22.047 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T02:58:22.047 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-03-07 02:45:51.634929621 +0000 2024-03-07T02:58:22.048 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-03-07 02:45:51.338932264 +0000 2024-03-07T02:58:22.048 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-03-07 02:45:51.338932264 +0000 2024-03-07T02:58:22.048 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: - 2024-03-07T02:58:22.048 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-07T02:58:22.106 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-03-07T02:58:22.106 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-03-07T02:58:22.106 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000370998 s, 1.4 MB/s 2024-03-07T02:58:22.107 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-07T02:58:22.161 INFO:tasks.cephadm:Deploying osd.0 on smithi145 with /dev/vg_nvme/lv_4... 2024-03-07T02:58:22.161 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-07T02:58:22.449 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:22 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:58:22.449 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:22 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:58:22.449 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:22 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:58:22.449 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:22 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:58:22.700 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:22 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:58:22.701 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:22 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:58:22.701 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:22 smithi189 systemd[1]: Started Ceph mgr.x for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T02:58:23.412 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:23 smithi189 bash[23614]: cluster 2024-03-07T02:58:21.240349+0000 mgr.y (mgr.14152) 76 : cluster 0 pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:23.412 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:23 smithi189 bash[23614]: audit 2024-03-07T02:58:22.700181+0000 mon.a (mon.0) 241 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:23.412 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:23 smithi189 bash[23614]: audit 2024-03-07T02:58:22.707734+0000 mon.a (mon.0) 242 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:23.412 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:23 smithi189 bash[23614]: audit 2024-03-07T02:58:22.730196+0000 mon.a (mon.0) 243 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:23.412 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:23 smithi189 bash[23614]: audit 2024-03-07T02:58:22.736987+0000 mon.a (mon.0) 244 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:23.412 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:23 smithi189 bash[23614]: audit 2024-03-07T02:58:22.806695+0000 mon.a (mon.0) 245 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:58:23.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:23 smithi145 bash[30906]: cluster 2024-03-07T02:58:21.240349+0000 mgr.y (mgr.14152) 76 : cluster 0 pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:23.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:23 smithi145 bash[30906]: audit 2024-03-07T02:58:22.700181+0000 mon.a (mon.0) 241 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:23.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:23 smithi145 bash[30906]: audit 2024-03-07T02:58:22.707734+0000 mon.a (mon.0) 242 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:23.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:23 smithi145 bash[30906]: audit 2024-03-07T02:58:22.730196+0000 mon.a (mon.0) 243 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:23.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:23 smithi145 bash[30906]: audit 2024-03-07T02:58:22.736987+0000 mon.a (mon.0) 244 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:23.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:23 smithi145 bash[30906]: audit 2024-03-07T02:58:22.806695+0000 mon.a (mon.0) 245 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:58:23.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:23 smithi145 bash[21339]: cluster 2024-03-07T02:58:21.240349+0000 mgr.y (mgr.14152) 76 : cluster 0 pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:23.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:23 smithi145 bash[21339]: audit 2024-03-07T02:58:22.700181+0000 mon.a (mon.0) 241 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:23.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:23 smithi145 bash[21339]: audit 2024-03-07T02:58:22.707734+0000 mon.a (mon.0) 242 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:23.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:23 smithi145 bash[21339]: audit 2024-03-07T02:58:22.730196+0000 mon.a (mon.0) 243 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:23.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:23 smithi145 bash[21339]: audit 2024-03-07T02:58:22.736987+0000 mon.a (mon.0) 244 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:23.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:23 smithi145 bash[21339]: audit 2024-03-07T02:58:22.806695+0000 mon.a (mon.0) 245 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:58:23.840 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:23 smithi189 bash[24563]: debug 2024-03-07T02:58:23.631+0000 7f7756bbb200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T02:58:25.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:25 smithi189 bash[23614]: cluster 2024-03-07T02:58:23.240739+0000 mgr.y (mgr.14152) 77 : cluster 0 pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:25.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:25 smithi189 bash[23614]: audit 2024-03-07T02:58:24.754546+0000 mon.a (mon.0) 246 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:25.525 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:25 smithi189 bash[24563]: debug 2024-03-07T02:58:25.179+0000 7f7756bbb200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T02:58:25.525 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:25 smithi189 bash[24563]: debug 2024-03-07T02:58:25.271+0000 7f7756bbb200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T02:58:25.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:25 smithi145 bash[21339]: cluster 2024-03-07T02:58:23.240739+0000 mgr.y (mgr.14152) 77 : cluster 0 pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:25.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:25 smithi145 bash[21339]: audit 2024-03-07T02:58:24.754546+0000 mon.a (mon.0) 246 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:25.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:25 smithi145 bash[30906]: cluster 2024-03-07T02:58:23.240739+0000 mgr.y (mgr.14152) 77 : cluster 0 pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:25.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:25 smithi145 bash[30906]: audit 2024-03-07T02:58:24.754546+0000 mon.a (mon.0) 246 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:26.590 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:25 smithi189 bash[24563]: debug 2024-03-07T02:58:25.523+0000 7f7756bbb200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T02:58:26.590 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:26 smithi189 bash[24563]: debug 2024-03-07T02:58:26.303+0000 7f7756bbb200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T02:58:26.590 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:26 smithi189 bash[24563]: debug 2024-03-07T02:58:26.411+0000 7f7756bbb200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T02:58:26.590 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:26 smithi189 bash[24563]: debug 2024-03-07T02:58:26.511+0000 7f7756bbb200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T02:58:26.834 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T02:58:27.168 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:26 smithi189 bash[24563]: debug 2024-03-07T02:58:26.859+0000 7f7756bbb200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T02:58:27.169 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:26 smithi189 bash[24563]: debug 2024-03-07T02:58:26.971+0000 7f7756bbb200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T02:58:27.169 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:27 smithi189 bash[24563]: debug 2024-03-07T02:58:27.071+0000 7f7756bbb200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T02:58:27.420 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:27 smithi189 bash[23614]: cluster 2024-03-07T02:58:25.241080+0000 mgr.y (mgr.14152) 78 : cluster 0 pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:27.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:27 smithi145 bash[21339]: cluster 2024-03-07T02:58:25.241080+0000 mgr.y (mgr.14152) 78 : cluster 0 pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:27.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:27 smithi145 bash[30906]: cluster 2024-03-07T02:58:25.241080+0000 mgr.y (mgr.14152) 78 : cluster 0 pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:27.840 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:27 smithi189 bash[24563]: debug 2024-03-07T02:58:27.419+0000 7f7756bbb200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T02:58:27.840 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:27 smithi189 bash[24563]: debug 2024-03-07T02:58:27.659+0000 7f7756bbb200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T02:58:28.298 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:27 smithi189 bash[24563]: debug 2024-03-07T02:58:27.919+0000 7f7756bbb200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T02:58:28.298 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[24563]: debug 2024-03-07T02:58:28.027+0000 7f7756bbb200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T02:58:28.590 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[24563]: debug 2024-03-07T02:58:28.295+0000 7f7756bbb200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T02:58:28.590 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[24563]: debug 2024-03-07T02:58:28.531+0000 7f7756bbb200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:58:29.292 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[24563]: debug 2024-03-07T02:58:28.939+0000 7f7756bbb200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T02:58:29.292 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:29 smithi189 bash[24563]: debug 2024-03-07T02:58:29.051+0000 7f7756bbb200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T02:58:29.292 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[23614]: cluster 2024-03-07T02:58:27.241473+0000 mgr.y (mgr.14152) 79 : cluster 0 pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:29.293 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[23614]: audit 2024-03-07T02:58:27.893636+0000 mon.a (mon.0) 247 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:29.293 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[23614]: audit 2024-03-07T02:58:27.897398+0000 mon.a (mon.0) 248 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:29.293 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[23614]: audit 2024-03-07T02:58:27.898281+0000 mon.a (mon.0) 249 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:29.293 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[23614]: audit 2024-03-07T02:58:27.899204+0000 mon.a (mon.0) 250 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:29.293 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[23614]: audit 2024-03-07T02:58:27.903720+0000 mon.a (mon.0) 251 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:29.293 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[23614]: cephadm 2024-03-07T02:58:27.953380+0000 mgr.y (mgr.14152) 80 : cephadm 1 Reconfiguring mgr.y (unknown last config time)... 2024-03-07T02:58:29.293 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[23614]: audit 2024-03-07T02:58:27.953947+0000 mon.a (mon.0) 252 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T02:58:29.293 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[23614]: audit 2024-03-07T02:58:27.955423+0000 mon.a (mon.0) 253 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T02:58:29.293 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[23614]: audit 2024-03-07T02:58:27.956366+0000 mon.a (mon.0) 254 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:29.293 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:28 smithi189 bash[23614]: cephadm 2024-03-07T02:58:27.957296+0000 mgr.y (mgr.14152) 81 : cephadm 1 Reconfiguring daemon mgr.y on smithi145 2024-03-07T02:58:29.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[21339]: cluster 2024-03-07T02:58:27.241473+0000 mgr.y (mgr.14152) 79 : cluster 0 pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:29.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[21339]: audit 2024-03-07T02:58:27.893636+0000 mon.a (mon.0) 247 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:29.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[21339]: audit 2024-03-07T02:58:27.897398+0000 mon.a (mon.0) 248 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:29.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[21339]: audit 2024-03-07T02:58:27.898281+0000 mon.a (mon.0) 249 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:29.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[21339]: audit 2024-03-07T02:58:27.899204+0000 mon.a (mon.0) 250 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:29.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[21339]: audit 2024-03-07T02:58:27.903720+0000 mon.a (mon.0) 251 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:29.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[21339]: cephadm 2024-03-07T02:58:27.953380+0000 mgr.y (mgr.14152) 80 : cephadm 1 Reconfiguring mgr.y (unknown last config time)... 2024-03-07T02:58:29.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[21339]: audit 2024-03-07T02:58:27.953947+0000 mon.a (mon.0) 252 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T02:58:29.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[21339]: audit 2024-03-07T02:58:27.955423+0000 mon.a (mon.0) 253 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T02:58:29.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[21339]: audit 2024-03-07T02:58:27.956366+0000 mon.a (mon.0) 254 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:29.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[21339]: cephadm 2024-03-07T02:58:27.957296+0000 mgr.y (mgr.14152) 81 : cephadm 1 Reconfiguring daemon mgr.y on smithi145 2024-03-07T02:58:29.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[30906]: cluster 2024-03-07T02:58:27.241473+0000 mgr.y (mgr.14152) 79 : cluster 0 pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:29.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[30906]: audit 2024-03-07T02:58:27.893636+0000 mon.a (mon.0) 247 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:29.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[30906]: audit 2024-03-07T02:58:27.897398+0000 mon.a (mon.0) 248 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:29.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[30906]: audit 2024-03-07T02:58:27.898281+0000 mon.a (mon.0) 249 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:29.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[30906]: audit 2024-03-07T02:58:27.899204+0000 mon.a (mon.0) 250 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:29.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[30906]: audit 2024-03-07T02:58:27.903720+0000 mon.a (mon.0) 251 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:29.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[30906]: cephadm 2024-03-07T02:58:27.953380+0000 mgr.y (mgr.14152) 80 : cephadm 1 Reconfiguring mgr.y (unknown last config time)... 2024-03-07T02:58:29.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[30906]: audit 2024-03-07T02:58:27.953947+0000 mon.a (mon.0) 252 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T02:58:29.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[30906]: audit 2024-03-07T02:58:27.955423+0000 mon.a (mon.0) 253 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T02:58:29.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[30906]: audit 2024-03-07T02:58:27.956366+0000 mon.a (mon.0) 254 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:29.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:28 smithi145 bash[30906]: cephadm 2024-03-07T02:58:27.957296+0000 mgr.y (mgr.14152) 81 : cephadm 1 Reconfiguring daemon mgr.y on smithi145 2024-03-07T02:58:30.590 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:29 smithi189 bash[24563]: debug 2024-03-07T02:58:29.291+0000 7f7756bbb200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:58:30.590 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:30 smithi189 bash[24563]: debug 2024-03-07T02:58:30.279+0000 7f7756bbb200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T02:58:30.590 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:30 smithi189 bash[24563]: debug 2024-03-07T02:58:30.391+0000 7f7756bbb200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T02:58:30.807 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-07T02:58:30.807 INFO:teuthology.orchestra.run.smithi145.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-07T02:58:30.807 INFO:teuthology.orchestra.run.smithi145.stderr: stderr: 10+0 records in 2024-03-07T02:58:30.807 INFO:teuthology.orchestra.run.smithi145.stderr:10+0 records out 2024-03-07T02:58:30.807 INFO:teuthology.orchestra.run.smithi145.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0153233 s, 684 MB/s 2024-03-07T02:58:30.807 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping successful for: 2024-03-07T02:58:31.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:30 smithi189 bash[23614]: cluster 2024-03-07T02:58:29.241825+0000 mgr.y (mgr.14152) 82 : cluster 0 pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:31.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:30 smithi189 bash[23614]: audit 2024-03-07T02:58:30.580287+0000 mon.a (mon.0) 255 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:31.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:30 smithi189 bash[23614]: audit 2024-03-07T02:58:30.584429+0000 mon.a (mon.0) 256 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:31.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:30 smithi189 bash[23614]: audit 2024-03-07T02:58:30.628655+0000 mon.a (mon.0) 257 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:58:31.090 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:30 smithi189 bash[24563]: debug 2024-03-07T02:58:30.675+0000 7f7756bbb200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T02:58:31.090 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:30 smithi189 bash[24563]: debug 2024-03-07T02:58:30.795+0000 7f7756bbb200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T02:58:31.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:30 smithi145 bash[21339]: cluster 2024-03-07T02:58:29.241825+0000 mgr.y (mgr.14152) 82 : cluster 0 pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:31.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:30 smithi145 bash[21339]: audit 2024-03-07T02:58:30.580287+0000 mon.a (mon.0) 255 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:31.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:30 smithi145 bash[21339]: audit 2024-03-07T02:58:30.584429+0000 mon.a (mon.0) 256 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:31.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:30 smithi145 bash[21339]: audit 2024-03-07T02:58:30.628655+0000 mon.a (mon.0) 257 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:58:31.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:30 smithi145 bash[30906]: cluster 2024-03-07T02:58:29.241825+0000 mgr.y (mgr.14152) 82 : cluster 0 pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:31.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:30 smithi145 bash[30906]: audit 2024-03-07T02:58:30.580287+0000 mon.a (mon.0) 255 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:31.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:30 smithi145 bash[30906]: audit 2024-03-07T02:58:30.584429+0000 mon.a (mon.0) 256 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:31.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:30 smithi145 bash[30906]: audit 2024-03-07T02:58:30.628655+0000 mon.a (mon.0) 257 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:58:31.473 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi145:vg_nvme/lv_4 2024-03-07T02:58:31.590 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:31 smithi189 bash[24563]: debug 2024-03-07T02:58:31.239+0000 7f7756bbb200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T02:58:31.590 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 02:58:31 smithi189 bash[24563]: debug 2024-03-07T02:58:31.355+0000 7f7756bbb200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T02:58:32.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[21339]: audit 2024-03-07T02:58:31.160914+0000 mon.a (mon.0) 258 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:32.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[21339]: audit 2024-03-07T02:58:31.162697+0000 mon.a (mon.0) 259 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:32.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[21339]: audit 2024-03-07T02:58:31.174637+0000 mon.a (mon.0) 260 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:32.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[21339]: cluster 2024-03-07T02:58:31.383100+0000 mon.a (mon.0) 261 : cluster 0 Standby manager daemon x started 2024-03-07T02:58:32.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[21339]: audit 2024-03-07T02:58:31.385055+0000 mon.b (mon.1) 4 : audit 0 from='mgr.? 172.21.15.189:0/3601295124' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T02:58:32.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[21339]: audit 2024-03-07T02:58:31.386005+0000 mon.b (mon.1) 5 : audit 0 from='mgr.? 172.21.15.189:0/3601295124' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T02:58:32.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[21339]: audit 2024-03-07T02:58:31.387593+0000 mon.b (mon.1) 6 : audit 0 from='mgr.? 172.21.15.189:0/3601295124' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T02:58:32.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[21339]: audit 2024-03-07T02:58:31.388308+0000 mon.b (mon.1) 7 : audit 0 from='mgr.? 172.21.15.189:0/3601295124' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T02:58:32.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[30906]: audit 2024-03-07T02:58:31.160914+0000 mon.a (mon.0) 258 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:32.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[30906]: audit 2024-03-07T02:58:31.162697+0000 mon.a (mon.0) 259 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:32.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[30906]: audit 2024-03-07T02:58:31.174637+0000 mon.a (mon.0) 260 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:32.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[30906]: cluster 2024-03-07T02:58:31.383100+0000 mon.a (mon.0) 261 : cluster 0 Standby manager daemon x started 2024-03-07T02:58:32.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[30906]: audit 2024-03-07T02:58:31.385055+0000 mon.b (mon.1) 4 : audit 0 from='mgr.? 172.21.15.189:0/3601295124' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T02:58:32.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[30906]: audit 2024-03-07T02:58:31.386005+0000 mon.b (mon.1) 5 : audit 0 from='mgr.? 172.21.15.189:0/3601295124' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T02:58:32.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[30906]: audit 2024-03-07T02:58:31.387593+0000 mon.b (mon.1) 6 : audit 0 from='mgr.? 172.21.15.189:0/3601295124' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T02:58:32.323 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:31 smithi145 bash[30906]: audit 2024-03-07T02:58:31.388308+0000 mon.b (mon.1) 7 : audit 0 from='mgr.? 172.21.15.189:0/3601295124' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T02:58:32.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:31 smithi189 bash[23614]: audit 2024-03-07T02:58:31.160914+0000 mon.a (mon.0) 258 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:32.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:31 smithi189 bash[23614]: audit 2024-03-07T02:58:31.162697+0000 mon.a (mon.0) 259 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:58:32.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:31 smithi189 bash[23614]: audit 2024-03-07T02:58:31.174637+0000 mon.a (mon.0) 260 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:58:32.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:31 smithi189 bash[23614]: cluster 2024-03-07T02:58:31.383100+0000 mon.a (mon.0) 261 : cluster 0 Standby manager daemon x started 2024-03-07T02:58:32.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:31 smithi189 bash[23614]: audit 2024-03-07T02:58:31.385055+0000 mon.b (mon.1) 4 : audit 0 from='mgr.? 172.21.15.189:0/3601295124' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T02:58:32.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:31 smithi189 bash[23614]: audit 2024-03-07T02:58:31.386005+0000 mon.b (mon.1) 5 : audit 0 from='mgr.? 172.21.15.189:0/3601295124' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T02:58:32.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:31 smithi189 bash[23614]: audit 2024-03-07T02:58:31.387593+0000 mon.b (mon.1) 6 : audit 0 from='mgr.? 172.21.15.189:0/3601295124' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T02:58:32.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:31 smithi189 bash[23614]: audit 2024-03-07T02:58:31.388308+0000 mon.b (mon.1) 7 : audit 0 from='mgr.? 172.21.15.189:0/3601295124' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T02:58:33.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:33 smithi145 bash[21339]: cluster 2024-03-07T02:58:31.242194+0000 mgr.y (mgr.14152) 83 : cluster 0 pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:33.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:33 smithi145 bash[21339]: cluster 2024-03-07T02:58:32.193228+0000 mon.a (mon.0) 262 : cluster 0 mgrmap e14: y(active, since 104s), standbys: x 2024-03-07T02:58:33.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:33 smithi145 bash[21339]: audit 2024-03-07T02:58:32.193538+0000 mon.a (mon.0) 263 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T02:58:33.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:33 smithi145 bash[30906]: cluster 2024-03-07T02:58:31.242194+0000 mgr.y (mgr.14152) 83 : cluster 0 pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:33.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:33 smithi145 bash[30906]: cluster 2024-03-07T02:58:32.193228+0000 mon.a (mon.0) 262 : cluster 0 mgrmap e14: y(active, since 104s), standbys: x 2024-03-07T02:58:33.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:33 smithi145 bash[30906]: audit 2024-03-07T02:58:32.193538+0000 mon.a (mon.0) 263 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T02:58:33.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:33 smithi189 bash[23614]: cluster 2024-03-07T02:58:31.242194+0000 mgr.y (mgr.14152) 83 : cluster 0 pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:33.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:33 smithi189 bash[23614]: cluster 2024-03-07T02:58:32.193228+0000 mon.a (mon.0) 262 : cluster 0 mgrmap e14: y(active, since 104s), standbys: x 2024-03-07T02:58:33.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:33 smithi189 bash[23614]: audit 2024-03-07T02:58:32.193538+0000 mon.a (mon.0) 263 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T02:58:35.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:35 smithi145 bash[21339]: cluster 2024-03-07T02:58:33.242524+0000 mgr.y (mgr.14152) 84 : cluster 0 pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:35.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:35 smithi145 bash[30906]: cluster 2024-03-07T02:58:33.242524+0000 mgr.y (mgr.14152) 84 : cluster 0 pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:35.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:35 smithi189 bash[23614]: cluster 2024-03-07T02:58:33.242524+0000 mgr.y (mgr.14152) 84 : cluster 0 pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:36.149 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T02:58:37.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:37 smithi145 bash[21339]: cluster 2024-03-07T02:58:35.242932+0000 mgr.y (mgr.14152) 85 : cluster 0 pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:37.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:37 smithi145 bash[30906]: cluster 2024-03-07T02:58:35.242932+0000 mgr.y (mgr.14152) 85 : cluster 0 pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:37.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:37 smithi189 bash[23614]: cluster 2024-03-07T02:58:35.242932+0000 mgr.y (mgr.14152) 85 : cluster 0 pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:39.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:39 smithi145 bash[21339]: cluster 2024-03-07T02:58:37.243341+0000 mgr.y (mgr.14152) 86 : cluster 0 pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:39.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:39 smithi145 bash[21339]: audit 2024-03-07T02:58:38.907384+0000 mon.a (mon.0) 264 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T02:58:39.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:39 smithi145 bash[21339]: audit 2024-03-07T02:58:38.911106+0000 mon.a (mon.0) 265 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T02:58:39.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:39 smithi145 bash[21339]: audit 2024-03-07T02:58:38.912105+0000 mon.a (mon.0) 266 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:39.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:39 smithi145 bash[30906]: cluster 2024-03-07T02:58:37.243341+0000 mgr.y (mgr.14152) 86 : cluster 0 pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:39.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:39 smithi145 bash[30906]: audit 2024-03-07T02:58:38.907384+0000 mon.a (mon.0) 264 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T02:58:39.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:39 smithi145 bash[30906]: audit 2024-03-07T02:58:38.911106+0000 mon.a (mon.0) 265 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T02:58:39.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:39 smithi145 bash[30906]: audit 2024-03-07T02:58:38.912105+0000 mon.a (mon.0) 266 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:39.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:39 smithi189 bash[23614]: cluster 2024-03-07T02:58:37.243341+0000 mgr.y (mgr.14152) 86 : cluster 0 pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:39.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:39 smithi189 bash[23614]: audit 2024-03-07T02:58:38.907384+0000 mon.a (mon.0) 264 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T02:58:39.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:39 smithi189 bash[23614]: audit 2024-03-07T02:58:38.911106+0000 mon.a (mon.0) 265 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T02:58:39.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:39 smithi189 bash[23614]: audit 2024-03-07T02:58:38.912105+0000 mon.a (mon.0) 266 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:40.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:40 smithi145 bash[21339]: audit 2024-03-07T02:58:38.902786+0000 mgr.y (mgr.14152) 87 : audit 0 from='client.14202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:58:40.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:40 smithi145 bash[30906]: audit 2024-03-07T02:58:38.902786+0000 mgr.y (mgr.14152) 87 : audit 0 from='client.14202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:58:40.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:40 smithi189 bash[23614]: audit 2024-03-07T02:58:38.902786+0000 mgr.y (mgr.14152) 87 : audit 0 from='client.14202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:58:41.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:41 smithi145 bash[21339]: cluster 2024-03-07T02:58:39.243690+0000 mgr.y (mgr.14152) 88 : cluster 0 pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:41 smithi145 bash[30906]: cluster 2024-03-07T02:58:39.243690+0000 mgr.y (mgr.14152) 88 : cluster 0 pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:41.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:41 smithi189 bash[23614]: cluster 2024-03-07T02:58:39.243690+0000 mgr.y (mgr.14152) 88 : cluster 0 pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:43.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:43 smithi145 bash[21339]: cluster 2024-03-07T02:58:41.244054+0000 mgr.y (mgr.14152) 89 : cluster 0 pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:43.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:43 smithi145 bash[30906]: cluster 2024-03-07T02:58:41.244054+0000 mgr.y (mgr.14152) 89 : cluster 0 pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:43.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:43 smithi189 bash[23614]: cluster 2024-03-07T02:58:41.244054+0000 mgr.y (mgr.14152) 89 : cluster 0 pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:45.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:45 smithi145 bash[21339]: cluster 2024-03-07T02:58:43.244494+0000 mgr.y (mgr.14152) 90 : cluster 0 pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:45.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:45 smithi145 bash[30906]: cluster 2024-03-07T02:58:43.244494+0000 mgr.y (mgr.14152) 90 : cluster 0 pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:45.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:45 smithi189 bash[23614]: cluster 2024-03-07T02:58:43.244494+0000 mgr.y (mgr.14152) 90 : cluster 0 pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:47.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:47 smithi145 bash[21339]: cluster 2024-03-07T02:58:45.244901+0000 mgr.y (mgr.14152) 91 : cluster 0 pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:47.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:47 smithi145 bash[21339]: audit 2024-03-07T02:58:47.020244+0000 mon.a (mon.0) 267 : audit 1 from='client.? 172.21.15.145:0/2661314610' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "242c63ee-0417-4ebf-bd71-6239f5d3634f"}]: dispatch 2024-03-07T02:58:47.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:47 smithi145 bash[21339]: audit 2024-03-07T02:58:47.027760+0000 mon.a (mon.0) 268 : audit 1 from='client.? 172.21.15.145:0/2661314610' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "242c63ee-0417-4ebf-bd71-6239f5d3634f"}]': finished 2024-03-07T02:58:47.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:47 smithi145 bash[21339]: cluster 2024-03-07T02:58:47.027797+0000 mon.a (mon.0) 269 : cluster 0 osdmap e5: 1 total, 0 up, 1 in 2024-03-07T02:58:47.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:47 smithi145 bash[21339]: audit 2024-03-07T02:58:47.028016+0000 mon.a (mon.0) 270 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:58:47.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:47 smithi145 bash[30906]: cluster 2024-03-07T02:58:45.244901+0000 mgr.y (mgr.14152) 91 : cluster 0 pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:47.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:47 smithi145 bash[30906]: audit 2024-03-07T02:58:47.020244+0000 mon.a (mon.0) 267 : audit 1 from='client.? 172.21.15.145:0/2661314610' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "242c63ee-0417-4ebf-bd71-6239f5d3634f"}]: dispatch 2024-03-07T02:58:47.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:47 smithi145 bash[30906]: audit 2024-03-07T02:58:47.027760+0000 mon.a (mon.0) 268 : audit 1 from='client.? 172.21.15.145:0/2661314610' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "242c63ee-0417-4ebf-bd71-6239f5d3634f"}]': finished 2024-03-07T02:58:47.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:47 smithi145 bash[30906]: cluster 2024-03-07T02:58:47.027797+0000 mon.a (mon.0) 269 : cluster 0 osdmap e5: 1 total, 0 up, 1 in 2024-03-07T02:58:47.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:47 smithi145 bash[30906]: audit 2024-03-07T02:58:47.028016+0000 mon.a (mon.0) 270 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:58:47.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:47 smithi189 bash[23614]: cluster 2024-03-07T02:58:45.244901+0000 mgr.y (mgr.14152) 91 : cluster 0 pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:47.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:47 smithi189 bash[23614]: audit 2024-03-07T02:58:47.020244+0000 mon.a (mon.0) 267 : audit 1 from='client.? 172.21.15.145:0/2661314610' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "242c63ee-0417-4ebf-bd71-6239f5d3634f"}]: dispatch 2024-03-07T02:58:47.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:47 smithi189 bash[23614]: audit 2024-03-07T02:58:47.027760+0000 mon.a (mon.0) 268 : audit 1 from='client.? 172.21.15.145:0/2661314610' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "242c63ee-0417-4ebf-bd71-6239f5d3634f"}]': finished 2024-03-07T02:58:47.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:47 smithi189 bash[23614]: cluster 2024-03-07T02:58:47.027797+0000 mon.a (mon.0) 269 : cluster 0 osdmap e5: 1 total, 0 up, 1 in 2024-03-07T02:58:47.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:47 smithi189 bash[23614]: audit 2024-03-07T02:58:47.028016+0000 mon.a (mon.0) 270 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:58:48.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:48 smithi145 bash[21339]: audit 2024-03-07T02:58:47.717435+0000 mon.c (mon.2) 3 : audit 0 from='client.? 172.21.15.145:0/1872566856' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T02:58:48.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:48 smithi145 bash[30906]: audit 2024-03-07T02:58:47.717435+0000 mon.c (mon.2) 3 : audit 0 from='client.? 172.21.15.145:0/1872566856' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T02:58:48.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:48 smithi189 bash[23614]: audit 2024-03-07T02:58:47.717435+0000 mon.c (mon.2) 3 : audit 0 from='client.? 172.21.15.145:0/1872566856' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T02:58:49.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:49 smithi145 bash[21339]: cluster 2024-03-07T02:58:47.245256+0000 mgr.y (mgr.14152) 92 : cluster 0 pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:49.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:49 smithi145 bash[30906]: cluster 2024-03-07T02:58:47.245256+0000 mgr.y (mgr.14152) 92 : cluster 0 pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:49 smithi189 bash[23614]: cluster 2024-03-07T02:58:47.245256+0000 mgr.y (mgr.14152) 92 : cluster 0 pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:51.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:51 smithi145 bash[21339]: cluster 2024-03-07T02:58:49.245642+0000 mgr.y (mgr.14152) 93 : cluster 0 pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:51.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:51 smithi145 bash[30906]: cluster 2024-03-07T02:58:49.245642+0000 mgr.y (mgr.14152) 93 : cluster 0 pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:51.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:51 smithi189 bash[23614]: cluster 2024-03-07T02:58:49.245642+0000 mgr.y (mgr.14152) 93 : cluster 0 pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:53.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:53 smithi145 bash[21339]: cluster 2024-03-07T02:58:51.245967+0000 mgr.y (mgr.14152) 94 : cluster 0 pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:53.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:53 smithi145 bash[30906]: cluster 2024-03-07T02:58:51.245967+0000 mgr.y (mgr.14152) 94 : cluster 0 pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:53.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:53 smithi189 bash[23614]: cluster 2024-03-07T02:58:51.245967+0000 mgr.y (mgr.14152) 94 : cluster 0 pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:54.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:54 smithi145 bash[21339]: cluster 2024-03-07T02:58:53.246349+0000 mgr.y (mgr.14152) 95 : cluster 0 pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:54.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:54 smithi145 bash[30906]: cluster 2024-03-07T02:58:53.246349+0000 mgr.y (mgr.14152) 95 : cluster 0 pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:54.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:54 smithi189 bash[23614]: cluster 2024-03-07T02:58:53.246349+0000 mgr.y (mgr.14152) 95 : cluster 0 pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:56.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:56 smithi145 bash[21339]: cluster 2024-03-07T02:58:55.246710+0000 mgr.y (mgr.14152) 96 : cluster 0 pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:56.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:56 smithi145 bash[30906]: cluster 2024-03-07T02:58:55.246710+0000 mgr.y (mgr.14152) 96 : cluster 0 pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:56.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:56 smithi189 bash[23614]: cluster 2024-03-07T02:58:55.246710+0000 mgr.y (mgr.14152) 96 : cluster 0 pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:59.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:58:59 smithi145 bash[21339]: cluster 2024-03-07T02:58:57.247134+0000 mgr.y (mgr.14152) 97 : cluster 0 pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:59.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:58:59 smithi145 bash[30906]: cluster 2024-03-07T02:58:57.247134+0000 mgr.y (mgr.14152) 97 : cluster 0 pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:58:59.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:58:59 smithi189 bash[23614]: cluster 2024-03-07T02:58:57.247134+0000 mgr.y (mgr.14152) 97 : cluster 0 pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:00.434 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:00 smithi145 bash[21339]: audit 2024-03-07T02:58:59.310677+0000 mon.a (mon.0) 271 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T02:59:00.434 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:00 smithi145 bash[21339]: audit 2024-03-07T02:58:59.312460+0000 mon.a (mon.0) 272 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:00.434 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:00 smithi145 bash[30906]: audit 2024-03-07T02:58:59.310677+0000 mon.a (mon.0) 271 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T02:59:00.434 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:00 smithi145 bash[30906]: audit 2024-03-07T02:58:59.312460+0000 mon.a (mon.0) 272 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:00.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:00 smithi189 bash[23614]: audit 2024-03-07T02:58:59.310677+0000 mon.a (mon.0) 271 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T02:59:00.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:00 smithi189 bash[23614]: audit 2024-03-07T02:58:59.312460+0000 mon.a (mon.0) 272 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:01.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:01 smithi145 bash[21339]: cluster 2024-03-07T02:58:59.247556+0000 mgr.y (mgr.14152) 98 : cluster 0 pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:01.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:01 smithi145 bash[21339]: cephadm 2024-03-07T02:58:59.313856+0000 mgr.y (mgr.14152) 99 : cephadm 1 Deploying daemon osd.0 on smithi145 2024-03-07T02:59:01.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:01 smithi145 bash[30906]: cluster 2024-03-07T02:58:59.247556+0000 mgr.y (mgr.14152) 98 : cluster 0 pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:01.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:01 smithi145 bash[30906]: cephadm 2024-03-07T02:58:59.313856+0000 mgr.y (mgr.14152) 99 : cephadm 1 Deploying daemon osd.0 on smithi145 2024-03-07T02:59:01.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:01 smithi189 bash[23614]: cluster 2024-03-07T02:58:59.247556+0000 mgr.y (mgr.14152) 98 : cluster 0 pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:01.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:01 smithi189 bash[23614]: cephadm 2024-03-07T02:58:59.313856+0000 mgr.y (mgr.14152) 99 : cephadm 1 Deploying daemon osd.0 on smithi145 2024-03-07T02:59:03.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:03 smithi145 bash[21339]: cluster 2024-03-07T02:59:01.247883+0000 mgr.y (mgr.14152) 100 : cluster 0 pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:03.375 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:03 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:03.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:03 smithi145 bash[30906]: cluster 2024-03-07T02:59:01.247883+0000 mgr.y (mgr.14152) 100 : cluster 0 pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:03.375 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:03 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:03.375 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:59:03 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:03.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:03 smithi189 bash[23614]: cluster 2024-03-07T02:59:01.247883+0000 mgr.y (mgr.14152) 100 : cluster 0 pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:03.656 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:03 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:03.657 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:59:03 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:03.657 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:03 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:04.509 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:04 smithi145 bash[21339]: audit 2024-03-07T02:59:03.639598+0000 mon.a (mon.0) 273 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:04.509 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:04 smithi145 bash[21339]: audit 2024-03-07T02:59:03.642547+0000 mon.a (mon.0) 274 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:04.509 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:04 smithi145 bash[21339]: audit 2024-03-07T02:59:03.649111+0000 mon.a (mon.0) 275 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:04.509 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:04 smithi145 bash[30906]: audit 2024-03-07T02:59:03.639598+0000 mon.a (mon.0) 273 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:04.509 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:04 smithi145 bash[30906]: audit 2024-03-07T02:59:03.642547+0000 mon.a (mon.0) 274 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:04.509 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:04 smithi145 bash[30906]: audit 2024-03-07T02:59:03.649111+0000 mon.a (mon.0) 275 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:04.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:04 smithi189 bash[23614]: audit 2024-03-07T02:59:03.639598+0000 mon.a (mon.0) 273 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:04.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:04 smithi189 bash[23614]: audit 2024-03-07T02:59:03.642547+0000 mon.a (mon.0) 274 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:04.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:04 smithi189 bash[23614]: audit 2024-03-07T02:59:03.649111+0000 mon.a (mon.0) 275 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:05.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:05 smithi189 bash[23614]: cluster 2024-03-07T02:59:03.248150+0000 mgr.y (mgr.14152) 101 : cluster 0 pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:05.821 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:05 smithi145 bash[21339]: cluster 2024-03-07T02:59:03.248150+0000 mgr.y (mgr.14152) 101 : cluster 0 pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:05.821 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:05 smithi145 bash[30906]: cluster 2024-03-07T02:59:03.248150+0000 mgr.y (mgr.14152) 101 : cluster 0 pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:07.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:07 smithi145 bash[21339]: cluster 2024-03-07T02:59:05.248497+0000 mgr.y (mgr.14152) 102 : cluster 0 pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:07.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:07 smithi145 bash[30906]: cluster 2024-03-07T02:59:05.248497+0000 mgr.y (mgr.14152) 102 : cluster 0 pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:07.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:07 smithi189 bash[23614]: cluster 2024-03-07T02:59:05.248497+0000 mgr.y (mgr.14152) 102 : cluster 0 pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:09.541 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:09 smithi145 bash[21339]: cluster 2024-03-07T02:59:07.248940+0000 mgr.y (mgr.14152) 103 : cluster 0 pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:09.542 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:09 smithi145 bash[21339]: audit 2024-03-07T02:59:08.771113+0000 mon.a (mon.0) 276 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:09.542 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:09 smithi145 bash[21339]: audit 2024-03-07T02:59:08.775412+0000 mon.a (mon.0) 277 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:09.542 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:09 smithi145 bash[30906]: cluster 2024-03-07T02:59:07.248940+0000 mgr.y (mgr.14152) 103 : cluster 0 pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:09.542 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:09 smithi145 bash[30906]: audit 2024-03-07T02:59:08.771113+0000 mon.a (mon.0) 276 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:09.542 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:09 smithi145 bash[30906]: audit 2024-03-07T02:59:08.775412+0000 mon.a (mon.0) 277 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:09.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:09 smithi189 bash[23614]: cluster 2024-03-07T02:59:07.248940+0000 mgr.y (mgr.14152) 103 : cluster 0 pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:09.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:09 smithi189 bash[23614]: audit 2024-03-07T02:59:08.771113+0000 mon.a (mon.0) 276 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:09.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:09 smithi189 bash[23614]: audit 2024-03-07T02:59:08.775412+0000 mon.a (mon.0) 277 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:10.448 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:10 smithi145 bash[21339]: audit 2024-03-07T02:59:09.246492+0000 mon.a (mon.0) 278 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:10.449 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:10 smithi145 bash[21339]: audit 2024-03-07T02:59:09.247414+0000 mon.a (mon.0) 279 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:59:10.449 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:10 smithi145 bash[21339]: audit 2024-03-07T02:59:09.253705+0000 mon.a (mon.0) 280 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:10.449 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:10 smithi145 bash[21339]: audit 2024-03-07T02:59:10.159197+0000 mon.a (mon.0) 281 : audit 1 from='osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T02:59:10.449 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:10 smithi145 bash[30906]: audit 2024-03-07T02:59:09.246492+0000 mon.a (mon.0) 278 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:10.449 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:10 smithi145 bash[30906]: audit 2024-03-07T02:59:09.247414+0000 mon.a (mon.0) 279 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:59:10.449 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:10 smithi145 bash[30906]: audit 2024-03-07T02:59:09.253705+0000 mon.a (mon.0) 280 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:10.449 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:10 smithi145 bash[30906]: audit 2024-03-07T02:59:10.159197+0000 mon.a (mon.0) 281 : audit 1 from='osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T02:59:10.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:10 smithi189 bash[23614]: audit 2024-03-07T02:59:09.246492+0000 mon.a (mon.0) 278 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:10.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:10 smithi189 bash[23614]: audit 2024-03-07T02:59:09.247414+0000 mon.a (mon.0) 279 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:59:10.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:10 smithi189 bash[23614]: audit 2024-03-07T02:59:09.253705+0000 mon.a (mon.0) 280 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:10.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:10 smithi189 bash[23614]: audit 2024-03-07T02:59:10.159197+0000 mon.a (mon.0) 281 : audit 1 from='osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T02:59:11.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:11 smithi145 bash[21339]: cluster 2024-03-07T02:59:09.249260+0000 mgr.y (mgr.14152) 104 : cluster 0 pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:11.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:11 smithi145 bash[21339]: audit 2024-03-07T02:59:10.257718+0000 mon.a (mon.0) 282 : audit 1 from='osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-07T02:59:11.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:11 smithi145 bash[21339]: cluster 2024-03-07T02:59:10.257852+0000 mon.a (mon.0) 283 : cluster 0 osdmap e6: 1 total, 0 up, 1 in 2024-03-07T02:59:11.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:11 smithi145 bash[21339]: audit 2024-03-07T02:59:10.258142+0000 mon.a (mon.0) 284 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:59:11.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:11 smithi145 bash[21339]: audit 2024-03-07T02:59:10.258966+0000 mon.a (mon.0) 285 : audit 1 from='osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T02:59:11.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:11 smithi145 bash[30906]: cluster 2024-03-07T02:59:09.249260+0000 mgr.y (mgr.14152) 104 : cluster 0 pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:11.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:11 smithi145 bash[30906]: audit 2024-03-07T02:59:10.257718+0000 mon.a (mon.0) 282 : audit 1 from='osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-07T02:59:11.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:11 smithi145 bash[30906]: cluster 2024-03-07T02:59:10.257852+0000 mon.a (mon.0) 283 : cluster 0 osdmap e6: 1 total, 0 up, 1 in 2024-03-07T02:59:11.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:11 smithi145 bash[30906]: audit 2024-03-07T02:59:10.258142+0000 mon.a (mon.0) 284 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:59:11.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:11 smithi145 bash[30906]: audit 2024-03-07T02:59:10.258966+0000 mon.a (mon.0) 285 : audit 1 from='osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T02:59:11.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:11 smithi189 bash[23614]: cluster 2024-03-07T02:59:09.249260+0000 mgr.y (mgr.14152) 104 : cluster 0 pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:11.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:11 smithi189 bash[23614]: audit 2024-03-07T02:59:10.257718+0000 mon.a (mon.0) 282 : audit 1 from='osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-07T02:59:11.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:11 smithi189 bash[23614]: cluster 2024-03-07T02:59:10.257852+0000 mon.a (mon.0) 283 : cluster 0 osdmap e6: 1 total, 0 up, 1 in 2024-03-07T02:59:11.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:11 smithi189 bash[23614]: audit 2024-03-07T02:59:10.258142+0000 mon.a (mon.0) 284 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:59:11.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:11 smithi189 bash[23614]: audit 2024-03-07T02:59:10.258966+0000 mon.a (mon.0) 285 : audit 1 from='osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T02:59:12.507 INFO:teuthology.orchestra.run.smithi145.stdout:Created osd(s) 0 on host 'smithi145' 2024-03-07T02:59:12.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:12 smithi145 bash[30906]: cluster 2024-03-07T02:59:11.249523+0000 mgr.y (mgr.14152) 105 : cluster 0 pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:12.525 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:12 smithi145 bash[30906]: audit 2024-03-07T02:59:11.257199+0000 mon.a (mon.0) 286 : audit 1 from='osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-03-07T02:59:12.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:12 smithi145 bash[30906]: cluster 2024-03-07T02:59:11.257258+0000 mon.a (mon.0) 287 : cluster 0 osdmap e7: 1 total, 0 up, 1 in 2024-03-07T02:59:12.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:12 smithi145 bash[30906]: audit 2024-03-07T02:59:11.257382+0000 mon.a (mon.0) 288 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:59:12.526 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:12 smithi145 bash[30906]: audit 2024-03-07T02:59:11.261728+0000 mon.a (mon.0) 289 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:59:12.526 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:12 smithi145 bash[21339]: cluster 2024-03-07T02:59:11.249523+0000 mgr.y (mgr.14152) 105 : cluster 0 pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:12.526 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:12 smithi145 bash[21339]: audit 2024-03-07T02:59:11.257199+0000 mon.a (mon.0) 286 : audit 1 from='osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-03-07T02:59:12.526 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:12 smithi145 bash[21339]: cluster 2024-03-07T02:59:11.257258+0000 mon.a (mon.0) 287 : cluster 0 osdmap e7: 1 total, 0 up, 1 in 2024-03-07T02:59:12.526 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:12 smithi145 bash[21339]: audit 2024-03-07T02:59:11.257382+0000 mon.a (mon.0) 288 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:59:12.526 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:12 smithi145 bash[21339]: audit 2024-03-07T02:59:11.261728+0000 mon.a (mon.0) 289 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:59:12.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:12 smithi189 bash[23614]: cluster 2024-03-07T02:59:11.249523+0000 mgr.y (mgr.14152) 105 : cluster 0 pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:12.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:12 smithi189 bash[23614]: audit 2024-03-07T02:59:11.257199+0000 mon.a (mon.0) 286 : audit 1 from='osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-03-07T02:59:12.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:12 smithi189 bash[23614]: cluster 2024-03-07T02:59:11.257258+0000 mon.a (mon.0) 287 : cluster 0 osdmap e7: 1 total, 0 up, 1 in 2024-03-07T02:59:12.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:12 smithi189 bash[23614]: audit 2024-03-07T02:59:11.257382+0000 mon.a (mon.0) 288 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:59:12.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:12 smithi189 bash[23614]: audit 2024-03-07T02:59:11.261728+0000 mon.a (mon.0) 289 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:59:13.212 DEBUG:teuthology.orchestra.run.smithi145:osd.0> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.0.service 2024-03-07T02:59:13.215 INFO:tasks.cephadm:Deploying osd.1 on smithi145 with /dev/vg_nvme/lv_3... 2024-03-07T02:59:13.215 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-07T02:59:13.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[21339]: cluster 2024-03-07T02:59:11.112734+0000 osd.0 (osd.0) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T02:59:13.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[21339]: cluster 2024-03-07T02:59:11.112887+0000 osd.0 (osd.0) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T02:59:13.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[21339]: cluster 2024-03-07T02:59:12.265442+0000 mon.a (mon.0) 290 : cluster 1 osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946] boot 2024-03-07T02:59:13.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[21339]: cluster 2024-03-07T02:59:12.265522+0000 mon.a (mon.0) 291 : cluster 0 osdmap e8: 1 total, 1 up, 1 in 2024-03-07T02:59:13.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[21339]: audit 2024-03-07T02:59:12.267649+0000 mon.a (mon.0) 292 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:59:13.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[21339]: audit 2024-03-07T02:59:12.455951+0000 mon.a (mon.0) 293 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:13.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[21339]: audit 2024-03-07T02:59:12.464805+0000 mon.a (mon.0) 294 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:13.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[21339]: audit 2024-03-07T02:59:12.475960+0000 mon.a (mon.0) 295 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:13.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[30906]: cluster 2024-03-07T02:59:11.112734+0000 osd.0 (osd.0) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T02:59:13.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[30906]: cluster 2024-03-07T02:59:11.112887+0000 osd.0 (osd.0) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T02:59:13.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[30906]: cluster 2024-03-07T02:59:12.265442+0000 mon.a (mon.0) 290 : cluster 1 osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946] boot 2024-03-07T02:59:13.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[30906]: cluster 2024-03-07T02:59:12.265522+0000 mon.a (mon.0) 291 : cluster 0 osdmap e8: 1 total, 1 up, 1 in 2024-03-07T02:59:13.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[30906]: audit 2024-03-07T02:59:12.267649+0000 mon.a (mon.0) 292 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:59:13.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[30906]: audit 2024-03-07T02:59:12.455951+0000 mon.a (mon.0) 293 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:13.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[30906]: audit 2024-03-07T02:59:12.464805+0000 mon.a (mon.0) 294 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:13.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:13 smithi145 bash[30906]: audit 2024-03-07T02:59:12.475960+0000 mon.a (mon.0) 295 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:13.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:13 smithi189 bash[23614]: cluster 2024-03-07T02:59:11.112734+0000 osd.0 (osd.0) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T02:59:13.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:13 smithi189 bash[23614]: cluster 2024-03-07T02:59:11.112887+0000 osd.0 (osd.0) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T02:59:13.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:13 smithi189 bash[23614]: cluster 2024-03-07T02:59:12.265442+0000 mon.a (mon.0) 290 : cluster 1 osd.0 [v2:172.21.15.145:6802/1367316946,v1:172.21.15.145:6803/1367316946] boot 2024-03-07T02:59:13.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:13 smithi189 bash[23614]: cluster 2024-03-07T02:59:12.265522+0000 mon.a (mon.0) 291 : cluster 0 osdmap e8: 1 total, 1 up, 1 in 2024-03-07T02:59:13.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:13 smithi189 bash[23614]: audit 2024-03-07T02:59:12.267649+0000 mon.a (mon.0) 292 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T02:59:13.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:13 smithi189 bash[23614]: audit 2024-03-07T02:59:12.455951+0000 mon.a (mon.0) 293 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:13.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:13 smithi189 bash[23614]: audit 2024-03-07T02:59:12.464805+0000 mon.a (mon.0) 294 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:13.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:13 smithi189 bash[23614]: audit 2024-03-07T02:59:12.475960+0000 mon.a (mon.0) 295 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:14.821 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:14 smithi145 bash[21339]: cluster 2024-03-07T02:59:13.249890+0000 mgr.y (mgr.14152) 106 : cluster 0 pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:14.821 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:14 smithi145 bash[21339]: cluster 2024-03-07T02:59:13.468133+0000 mon.a (mon.0) 296 : cluster 0 osdmap e9: 1 total, 1 up, 1 in 2024-03-07T02:59:14.821 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:14 smithi145 bash[30906]: cluster 2024-03-07T02:59:13.249890+0000 mgr.y (mgr.14152) 106 : cluster 0 pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:14.821 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:14 smithi145 bash[30906]: cluster 2024-03-07T02:59:13.468133+0000 mon.a (mon.0) 296 : cluster 0 osdmap e9: 1 total, 1 up, 1 in 2024-03-07T02:59:14.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:14 smithi189 bash[23614]: cluster 2024-03-07T02:59:13.249890+0000 mgr.y (mgr.14152) 106 : cluster 0 pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:14.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:14 smithi189 bash[23614]: cluster 2024-03-07T02:59:13.468133+0000 mon.a (mon.0) 296 : cluster 0 osdmap e9: 1 total, 1 up, 1 in 2024-03-07T02:59:16.821 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:16 smithi145 bash[21339]: cluster 2024-03-07T02:59:15.250405+0000 mgr.y (mgr.14152) 107 : cluster 0 pgmap v73: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:16.821 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:16 smithi145 bash[30906]: cluster 2024-03-07T02:59:15.250405+0000 mgr.y (mgr.14152) 107 : cluster 0 pgmap v73: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:16.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:16 smithi189 bash[23614]: cluster 2024-03-07T02:59:15.250405+0000 mgr.y (mgr.14152) 107 : cluster 0 pgmap v73: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:17.304 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T02:59:19.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:19 smithi145 bash[21339]: cluster 2024-03-07T02:59:17.250887+0000 mgr.y (mgr.14152) 108 : cluster 0 pgmap v74: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:19.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:19 smithi145 bash[30906]: cluster 2024-03-07T02:59:17.250887+0000 mgr.y (mgr.14152) 108 : cluster 0 pgmap v74: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:19.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:19 smithi189 bash[23614]: cluster 2024-03-07T02:59:17.250887+0000 mgr.y (mgr.14152) 108 : cluster 0 pgmap v74: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:21.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:21 smithi145 bash[21339]: cluster 2024-03-07T02:59:19.251317+0000 mgr.y (mgr.14152) 109 : cluster 0 pgmap v75: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:21.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:21 smithi145 bash[30906]: cluster 2024-03-07T02:59:19.251317+0000 mgr.y (mgr.14152) 109 : cluster 0 pgmap v75: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:21.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:21 smithi189 bash[23614]: cluster 2024-03-07T02:59:19.251317+0000 mgr.y (mgr.14152) 109 : cluster 0 pgmap v75: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:22.020 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-07T02:59:22.020 INFO:teuthology.orchestra.run.smithi145.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-07T02:59:22.020 INFO:teuthology.orchestra.run.smithi145.stderr: stderr: 10+0 records in 2024-03-07T02:59:22.020 INFO:teuthology.orchestra.run.smithi145.stderr:10+0 records out 2024-03-07T02:59:22.021 INFO:teuthology.orchestra.run.smithi145.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.014626 s, 717 MB/s 2024-03-07T02:59:22.021 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping successful for: 2024-03-07T02:59:22.950 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi145:vg_nvme/lv_3 2024-03-07T02:59:23.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:23 smithi145 bash[30906]: cluster 2024-03-07T02:59:21.251730+0000 mgr.y (mgr.14152) 110 : cluster 0 pgmap v76: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:23.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:23 smithi145 bash[21339]: cluster 2024-03-07T02:59:21.251730+0000 mgr.y (mgr.14152) 110 : cluster 0 pgmap v76: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:23.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:23 smithi189 bash[23614]: cluster 2024-03-07T02:59:21.251730+0000 mgr.y (mgr.14152) 110 : cluster 0 pgmap v76: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:24.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:24 smithi189 bash[23614]: cluster 2024-03-07T02:59:23.252180+0000 mgr.y (mgr.14152) 111 : cluster 0 pgmap v77: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:24.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:24 smithi189 bash[23614]: cephadm 2024-03-07T02:59:23.318832+0000 mgr.y (mgr.14152) 112 : cephadm 1 Detected new or changed devices on smithi145 2024-03-07T02:59:24.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:24 smithi189 bash[23614]: audit 2024-03-07T02:59:23.327560+0000 mon.a (mon.0) 297 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:24.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:24 smithi189 bash[23614]: audit 2024-03-07T02:59:23.336244+0000 mon.a (mon.0) 298 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:24.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:24 smithi189 bash[23614]: audit 2024-03-07T02:59:23.338333+0000 mon.a (mon.0) 299 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T02:59:24.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:24 smithi189 bash[23614]: cephadm 2024-03-07T02:59:23.339570+0000 mgr.y (mgr.14152) 113 : cephadm 1 Adjusting osd_memory_target on smithi145 to 16241M 2024-03-07T02:59:24.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:24 smithi189 bash[23614]: audit 2024-03-07T02:59:23.347738+0000 mon.a (mon.0) 300 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:24.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:24 smithi189 bash[23614]: audit 2024-03-07T02:59:23.349960+0000 mon.a (mon.0) 301 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:24.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:24 smithi189 bash[23614]: audit 2024-03-07T02:59:23.351712+0000 mon.a (mon.0) 302 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:59:24.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:24 smithi189 bash[23614]: audit 2024-03-07T02:59:23.361685+0000 mon.a (mon.0) 303 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:24.821 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[21339]: cluster 2024-03-07T02:59:23.252180+0000 mgr.y (mgr.14152) 111 : cluster 0 pgmap v77: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:24.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[21339]: cephadm 2024-03-07T02:59:23.318832+0000 mgr.y (mgr.14152) 112 : cephadm 1 Detected new or changed devices on smithi145 2024-03-07T02:59:24.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[21339]: audit 2024-03-07T02:59:23.327560+0000 mon.a (mon.0) 297 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:24.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[21339]: audit 2024-03-07T02:59:23.336244+0000 mon.a (mon.0) 298 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:24.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[21339]: audit 2024-03-07T02:59:23.338333+0000 mon.a (mon.0) 299 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T02:59:24.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[21339]: cephadm 2024-03-07T02:59:23.339570+0000 mgr.y (mgr.14152) 113 : cephadm 1 Adjusting osd_memory_target on smithi145 to 16241M 2024-03-07T02:59:24.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[21339]: audit 2024-03-07T02:59:23.347738+0000 mon.a (mon.0) 300 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:24.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[21339]: audit 2024-03-07T02:59:23.349960+0000 mon.a (mon.0) 301 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:24.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[21339]: audit 2024-03-07T02:59:23.351712+0000 mon.a (mon.0) 302 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:59:24.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[21339]: audit 2024-03-07T02:59:23.361685+0000 mon.a (mon.0) 303 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:24.822 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[30906]: cluster 2024-03-07T02:59:23.252180+0000 mgr.y (mgr.14152) 111 : cluster 0 pgmap v77: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:24.822 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[30906]: cephadm 2024-03-07T02:59:23.318832+0000 mgr.y (mgr.14152) 112 : cephadm 1 Detected new or changed devices on smithi145 2024-03-07T02:59:24.822 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[30906]: audit 2024-03-07T02:59:23.327560+0000 mon.a (mon.0) 297 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:24.822 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[30906]: audit 2024-03-07T02:59:23.336244+0000 mon.a (mon.0) 298 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:24.823 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[30906]: audit 2024-03-07T02:59:23.338333+0000 mon.a (mon.0) 299 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T02:59:24.823 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[30906]: cephadm 2024-03-07T02:59:23.339570+0000 mgr.y (mgr.14152) 113 : cephadm 1 Adjusting osd_memory_target on smithi145 to 16241M 2024-03-07T02:59:24.823 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[30906]: audit 2024-03-07T02:59:23.347738+0000 mon.a (mon.0) 300 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:24.823 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[30906]: audit 2024-03-07T02:59:23.349960+0000 mon.a (mon.0) 301 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:24.823 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[30906]: audit 2024-03-07T02:59:23.351712+0000 mon.a (mon.0) 302 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T02:59:24.823 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:24 smithi145 bash[30906]: audit 2024-03-07T02:59:23.361685+0000 mon.a (mon.0) 303 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:26.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:26 smithi189 bash[23614]: cluster 2024-03-07T02:59:25.252639+0000 mgr.y (mgr.14152) 114 : cluster 0 pgmap v78: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:26.821 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:26 smithi145 bash[21339]: cluster 2024-03-07T02:59:25.252639+0000 mgr.y (mgr.14152) 114 : cluster 0 pgmap v78: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:26.821 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:26 smithi145 bash[30906]: cluster 2024-03-07T02:59:25.252639+0000 mgr.y (mgr.14152) 114 : cluster 0 pgmap v78: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:27.637 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T02:59:29.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:29 smithi145 bash[21339]: cluster 2024-03-07T02:59:27.253041+0000 mgr.y (mgr.14152) 115 : cluster 0 pgmap v79: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:29.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:29 smithi145 bash[30906]: cluster 2024-03-07T02:59:27.253041+0000 mgr.y (mgr.14152) 115 : cluster 0 pgmap v79: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:29.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:29 smithi189 bash[23614]: cluster 2024-03-07T02:59:27.253041+0000 mgr.y (mgr.14152) 115 : cluster 0 pgmap v79: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:31.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:31 smithi145 bash[21339]: cluster 2024-03-07T02:59:29.253462+0000 mgr.y (mgr.14152) 116 : cluster 0 pgmap v80: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:31.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:31 smithi145 bash[21339]: audit 2024-03-07T02:59:30.494217+0000 mon.a (mon.0) 304 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T02:59:31.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:31 smithi145 bash[21339]: audit 2024-03-07T02:59:30.498110+0000 mon.a (mon.0) 305 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T02:59:31.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:31 smithi145 bash[21339]: audit 2024-03-07T02:59:30.499190+0000 mon.a (mon.0) 306 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:31.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:31 smithi145 bash[30906]: cluster 2024-03-07T02:59:29.253462+0000 mgr.y (mgr.14152) 116 : cluster 0 pgmap v80: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:31.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:31 smithi145 bash[30906]: audit 2024-03-07T02:59:30.494217+0000 mon.a (mon.0) 304 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T02:59:31.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:31 smithi145 bash[30906]: audit 2024-03-07T02:59:30.498110+0000 mon.a (mon.0) 305 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T02:59:31.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:31 smithi145 bash[30906]: audit 2024-03-07T02:59:30.499190+0000 mon.a (mon.0) 306 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:31.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:31 smithi189 bash[23614]: cluster 2024-03-07T02:59:29.253462+0000 mgr.y (mgr.14152) 116 : cluster 0 pgmap v80: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:31.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:31 smithi189 bash[23614]: audit 2024-03-07T02:59:30.494217+0000 mon.a (mon.0) 304 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T02:59:31.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:31 smithi189 bash[23614]: audit 2024-03-07T02:59:30.498110+0000 mon.a (mon.0) 305 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T02:59:31.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:31 smithi189 bash[23614]: audit 2024-03-07T02:59:30.499190+0000 mon.a (mon.0) 306 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:32.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:32 smithi145 bash[21339]: audit 2024-03-07T02:59:30.489809+0000 mgr.y (mgr.14152) 117 : audit 0 from='client.14229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:59:32.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:32 smithi145 bash[30906]: audit 2024-03-07T02:59:30.489809+0000 mgr.y (mgr.14152) 117 : audit 0 from='client.14229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:59:32.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:32 smithi189 bash[23614]: audit 2024-03-07T02:59:30.489809+0000 mgr.y (mgr.14152) 117 : audit 0 from='client.14229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:59:33.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:33 smithi145 bash[21339]: cluster 2024-03-07T02:59:31.253898+0000 mgr.y (mgr.14152) 118 : cluster 0 pgmap v81: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:33.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:33 smithi145 bash[30906]: cluster 2024-03-07T02:59:31.253898+0000 mgr.y (mgr.14152) 118 : cluster 0 pgmap v81: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:33.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:33 smithi189 bash[23614]: cluster 2024-03-07T02:59:31.253898+0000 mgr.y (mgr.14152) 118 : cluster 0 pgmap v81: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:35.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:35 smithi145 bash[21339]: cluster 2024-03-07T02:59:33.254353+0000 mgr.y (mgr.14152) 119 : cluster 0 pgmap v82: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:35.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:35 smithi145 bash[30906]: cluster 2024-03-07T02:59:33.254353+0000 mgr.y (mgr.14152) 119 : cluster 0 pgmap v82: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:35.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:35 smithi189 bash[23614]: cluster 2024-03-07T02:59:33.254353+0000 mgr.y (mgr.14152) 119 : cluster 0 pgmap v82: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:37.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:37 smithi145 bash[21339]: cluster 2024-03-07T02:59:35.254764+0000 mgr.y (mgr.14152) 120 : cluster 0 pgmap v83: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:37.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:37 smithi145 bash[30906]: cluster 2024-03-07T02:59:35.254764+0000 mgr.y (mgr.14152) 120 : cluster 0 pgmap v83: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:37.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:37 smithi189 bash[23614]: cluster 2024-03-07T02:59:35.254764+0000 mgr.y (mgr.14152) 120 : cluster 0 pgmap v83: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:39.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:39 smithi145 bash[21339]: cluster 2024-03-07T02:59:37.255231+0000 mgr.y (mgr.14152) 121 : cluster 0 pgmap v84: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:39.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:39 smithi145 bash[21339]: audit 2024-03-07T02:59:38.694825+0000 mon.c (mon.2) 4 : audit 1 from='client.? 172.21.15.145:0/3993011627' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3c2818da-5d32-4f7a-8a63-8d7d679961e8"}]: dispatch 2024-03-07T02:59:39.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:39 smithi145 bash[21339]: audit 2024-03-07T02:59:38.695368+0000 mon.a (mon.0) 307 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3c2818da-5d32-4f7a-8a63-8d7d679961e8"}]: dispatch 2024-03-07T02:59:39.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:39 smithi145 bash[21339]: audit 2024-03-07T02:59:38.704125+0000 mon.a (mon.0) 308 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3c2818da-5d32-4f7a-8a63-8d7d679961e8"}]': finished 2024-03-07T02:59:39.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:39 smithi145 bash[21339]: cluster 2024-03-07T02:59:38.704314+0000 mon.a (mon.0) 309 : cluster 0 osdmap e10: 2 total, 1 up, 2 in 2024-03-07T02:59:39.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:39 smithi145 bash[21339]: audit 2024-03-07T02:59:38.704456+0000 mon.a (mon.0) 310 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T02:59:39.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:39 smithi145 bash[30906]: cluster 2024-03-07T02:59:37.255231+0000 mgr.y (mgr.14152) 121 : cluster 0 pgmap v84: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:39.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:39 smithi145 bash[30906]: audit 2024-03-07T02:59:38.694825+0000 mon.c (mon.2) 4 : audit 1 from='client.? 172.21.15.145:0/3993011627' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3c2818da-5d32-4f7a-8a63-8d7d679961e8"}]: dispatch 2024-03-07T02:59:39.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:39 smithi145 bash[30906]: audit 2024-03-07T02:59:38.695368+0000 mon.a (mon.0) 307 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3c2818da-5d32-4f7a-8a63-8d7d679961e8"}]: dispatch 2024-03-07T02:59:39.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:39 smithi145 bash[30906]: audit 2024-03-07T02:59:38.704125+0000 mon.a (mon.0) 308 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3c2818da-5d32-4f7a-8a63-8d7d679961e8"}]': finished 2024-03-07T02:59:39.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:39 smithi145 bash[30906]: cluster 2024-03-07T02:59:38.704314+0000 mon.a (mon.0) 309 : cluster 0 osdmap e10: 2 total, 1 up, 2 in 2024-03-07T02:59:39.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:39 smithi145 bash[30906]: audit 2024-03-07T02:59:38.704456+0000 mon.a (mon.0) 310 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T02:59:39.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:39 smithi189 bash[23614]: cluster 2024-03-07T02:59:37.255231+0000 mgr.y (mgr.14152) 121 : cluster 0 pgmap v84: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:39.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:39 smithi189 bash[23614]: audit 2024-03-07T02:59:38.694825+0000 mon.c (mon.2) 4 : audit 1 from='client.? 172.21.15.145:0/3993011627' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3c2818da-5d32-4f7a-8a63-8d7d679961e8"}]: dispatch 2024-03-07T02:59:39.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:39 smithi189 bash[23614]: audit 2024-03-07T02:59:38.695368+0000 mon.a (mon.0) 307 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3c2818da-5d32-4f7a-8a63-8d7d679961e8"}]: dispatch 2024-03-07T02:59:39.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:39 smithi189 bash[23614]: audit 2024-03-07T02:59:38.704125+0000 mon.a (mon.0) 308 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3c2818da-5d32-4f7a-8a63-8d7d679961e8"}]': finished 2024-03-07T02:59:39.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:39 smithi189 bash[23614]: cluster 2024-03-07T02:59:38.704314+0000 mon.a (mon.0) 309 : cluster 0 osdmap e10: 2 total, 1 up, 2 in 2024-03-07T02:59:39.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:39 smithi189 bash[23614]: audit 2024-03-07T02:59:38.704456+0000 mon.a (mon.0) 310 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T02:59:40.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:40 smithi145 bash[21339]: audit 2024-03-07T02:59:39.442392+0000 mon.a (mon.0) 311 : audit 0 from='client.? 172.21.15.145:0/2844069439' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T02:59:40.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:40 smithi145 bash[30906]: audit 2024-03-07T02:59:39.442392+0000 mon.a (mon.0) 311 : audit 0 from='client.? 172.21.15.145:0/2844069439' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T02:59:40.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:40 smithi189 bash[23614]: audit 2024-03-07T02:59:39.442392+0000 mon.a (mon.0) 311 : audit 0 from='client.? 172.21.15.145:0/2844069439' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T02:59:41.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:41 smithi145 bash[21339]: cluster 2024-03-07T02:59:39.255639+0000 mgr.y (mgr.14152) 122 : cluster 0 pgmap v86: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:41 smithi145 bash[30906]: cluster 2024-03-07T02:59:39.255639+0000 mgr.y (mgr.14152) 122 : cluster 0 pgmap v86: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:41.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:41 smithi189 bash[23614]: cluster 2024-03-07T02:59:39.255639+0000 mgr.y (mgr.14152) 122 : cluster 0 pgmap v86: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:43.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:43 smithi145 bash[21339]: cluster 2024-03-07T02:59:41.256124+0000 mgr.y (mgr.14152) 123 : cluster 0 pgmap v87: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:43.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:43 smithi145 bash[30906]: cluster 2024-03-07T02:59:41.256124+0000 mgr.y (mgr.14152) 123 : cluster 0 pgmap v87: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:43.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:43 smithi189 bash[23614]: cluster 2024-03-07T02:59:41.256124+0000 mgr.y (mgr.14152) 123 : cluster 0 pgmap v87: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:45.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:45 smithi145 bash[21339]: cluster 2024-03-07T02:59:43.256610+0000 mgr.y (mgr.14152) 124 : cluster 0 pgmap v88: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:45.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:45 smithi145 bash[30906]: cluster 2024-03-07T02:59:43.256610+0000 mgr.y (mgr.14152) 124 : cluster 0 pgmap v88: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:45.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:45 smithi189 bash[23614]: cluster 2024-03-07T02:59:43.256610+0000 mgr.y (mgr.14152) 124 : cluster 0 pgmap v88: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:47.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:47 smithi145 bash[21339]: cluster 2024-03-07T02:59:45.257064+0000 mgr.y (mgr.14152) 125 : cluster 0 pgmap v89: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:47.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:47 smithi145 bash[30906]: cluster 2024-03-07T02:59:45.257064+0000 mgr.y (mgr.14152) 125 : cluster 0 pgmap v89: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:47.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:47 smithi189 bash[23614]: cluster 2024-03-07T02:59:45.257064+0000 mgr.y (mgr.14152) 125 : cluster 0 pgmap v89: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:49.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:49 smithi145 bash[21339]: cluster 2024-03-07T02:59:47.257502+0000 mgr.y (mgr.14152) 126 : cluster 0 pgmap v90: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:49.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:49 smithi145 bash[30906]: cluster 2024-03-07T02:59:47.257502+0000 mgr.y (mgr.14152) 126 : cluster 0 pgmap v90: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:49 smithi189 bash[23614]: cluster 2024-03-07T02:59:47.257502+0000 mgr.y (mgr.14152) 126 : cluster 0 pgmap v90: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:51.510 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:51 smithi145 bash[21339]: cluster 2024-03-07T02:59:49.257838+0000 mgr.y (mgr.14152) 127 : cluster 0 pgmap v91: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:51.510 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:51 smithi145 bash[21339]: audit 2024-03-07T02:59:51.238523+0000 mon.a (mon.0) 312 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T02:59:51.511 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:51 smithi145 bash[30906]: cluster 2024-03-07T02:59:49.257838+0000 mgr.y (mgr.14152) 127 : cluster 0 pgmap v91: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:51.511 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:51 smithi145 bash[30906]: audit 2024-03-07T02:59:51.238523+0000 mon.a (mon.0) 312 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T02:59:51.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:51 smithi189 bash[23614]: cluster 2024-03-07T02:59:49.257838+0000 mgr.y (mgr.14152) 127 : cluster 0 pgmap v91: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:51.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:51 smithi189 bash[23614]: audit 2024-03-07T02:59:51.238523+0000 mon.a (mon.0) 312 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T02:59:52.513 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:52 smithi145 bash[21339]: audit 2024-03-07T02:59:51.240685+0000 mon.a (mon.0) 313 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:52.513 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:52 smithi145 bash[21339]: cephadm 2024-03-07T02:59:51.242262+0000 mgr.y (mgr.14152) 128 : cephadm 1 Deploying daemon osd.1 on smithi145 2024-03-07T02:59:52.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:52 smithi145 bash[30906]: audit 2024-03-07T02:59:51.240685+0000 mon.a (mon.0) 313 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:52.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:52 smithi145 bash[30906]: cephadm 2024-03-07T02:59:51.242262+0000 mgr.y (mgr.14152) 128 : cephadm 1 Deploying daemon osd.1 on smithi145 2024-03-07T02:59:52.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:52 smithi189 bash[23614]: audit 2024-03-07T02:59:51.240685+0000 mon.a (mon.0) 313 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:59:52.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:52 smithi189 bash[23614]: cephadm 2024-03-07T02:59:51.242262+0000 mgr.y (mgr.14152) 128 : cephadm 1 Deploying daemon osd.1 on smithi145 2024-03-07T02:59:53.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:53 smithi145 bash[21339]: cluster 2024-03-07T02:59:51.258301+0000 mgr.y (mgr.14152) 129 : cluster 0 pgmap v92: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:53.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:53 smithi145 bash[30906]: cluster 2024-03-07T02:59:51.258301+0000 mgr.y (mgr.14152) 129 : cluster 0 pgmap v92: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:53.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:53 smithi189 bash[23614]: cluster 2024-03-07T02:59:51.258301+0000 mgr.y (mgr.14152) 129 : cluster 0 pgmap v92: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:54.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:54 smithi145 bash[21339]: cluster 2024-03-07T02:59:53.258778+0000 mgr.y (mgr.14152) 130 : cluster 0 pgmap v93: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:54.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:54 smithi145 bash[30906]: cluster 2024-03-07T02:59:53.258778+0000 mgr.y (mgr.14152) 130 : cluster 0 pgmap v93: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:54.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:54 smithi189 bash[23614]: cluster 2024-03-07T02:59:53.258778+0000 mgr.y (mgr.14152) 130 : cluster 0 pgmap v93: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:55.753 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:55 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:55.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:59:55 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:55.754 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:55 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:55.754 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 02:59:55 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:56.044 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:55 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:56.044 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 02:59:55 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:56.044 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 02:59:55 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:56.044 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:55 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:59:57.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:57 smithi145 bash[21339]: cluster 2024-03-07T02:59:55.259286+0000 mgr.y (mgr.14152) 131 : cluster 0 pgmap v94: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:57.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:57 smithi145 bash[21339]: audit 2024-03-07T02:59:56.034700+0000 mon.a (mon.0) 314 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:57.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:57 smithi145 bash[21339]: audit 2024-03-07T02:59:56.040553+0000 mon.a (mon.0) 315 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:57.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:57 smithi145 bash[21339]: audit 2024-03-07T02:59:56.074514+0000 mon.a (mon.0) 316 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:57.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:57 smithi145 bash[30906]: cluster 2024-03-07T02:59:55.259286+0000 mgr.y (mgr.14152) 131 : cluster 0 pgmap v94: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:57.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:57 smithi145 bash[30906]: audit 2024-03-07T02:59:56.034700+0000 mon.a (mon.0) 314 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:57.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:57 smithi145 bash[30906]: audit 2024-03-07T02:59:56.040553+0000 mon.a (mon.0) 315 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:57.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:57 smithi145 bash[30906]: audit 2024-03-07T02:59:56.074514+0000 mon.a (mon.0) 316 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:57.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:57 smithi189 bash[23614]: cluster 2024-03-07T02:59:55.259286+0000 mgr.y (mgr.14152) 131 : cluster 0 pgmap v94: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:57.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:57 smithi189 bash[23614]: audit 2024-03-07T02:59:56.034700+0000 mon.a (mon.0) 314 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:57.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:57 smithi189 bash[23614]: audit 2024-03-07T02:59:56.040553+0000 mon.a (mon.0) 315 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T02:59:57.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:57 smithi189 bash[23614]: audit 2024-03-07T02:59:56.074514+0000 mon.a (mon.0) 316 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:59.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 02:59:59 smithi145 bash[21339]: cluster 2024-03-07T02:59:57.259599+0000 mgr.y (mgr.14152) 132 : cluster 0 pgmap v95: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:59.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 02:59:59 smithi145 bash[30906]: cluster 2024-03-07T02:59:57.259599+0000 mgr.y (mgr.14152) 132 : cluster 0 pgmap v95: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T02:59:59.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 02:59:59 smithi189 bash[23614]: cluster 2024-03-07T02:59:57.259599+0000 mgr.y (mgr.14152) 132 : cluster 0 pgmap v95: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:00:00.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:00 smithi145 bash[21339]: cluster 2024-03-07T03:00:00.000159+0000 mon.a (mon.0) 317 : cluster 1 overall HEALTH_OK 2024-03-07T03:00:00.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:00 smithi145 bash[30906]: cluster 2024-03-07T03:00:00.000159+0000 mon.a (mon.0) 317 : cluster 1 overall HEALTH_OK 2024-03-07T03:00:00.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:00 smithi189 bash[23614]: cluster 2024-03-07T03:00:00.000159+0000 mon.a (mon.0) 317 : cluster 1 overall HEALTH_OK 2024-03-07T03:00:01.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:01 smithi145 bash[21339]: cluster 2024-03-07T02:59:59.260057+0000 mgr.y (mgr.14152) 133 : cluster 0 pgmap v96: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:00:01.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:01 smithi145 bash[30906]: cluster 2024-03-07T02:59:59.260057+0000 mgr.y (mgr.14152) 133 : cluster 0 pgmap v96: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:00:01.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:01 smithi189 bash[23614]: cluster 2024-03-07T02:59:59.260057+0000 mgr.y (mgr.14152) 133 : cluster 0 pgmap v96: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:00:02.547 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:02 smithi145 bash[21339]: audit 2024-03-07T03:00:01.215093+0000 mon.a (mon.0) 318 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:02.547 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:02 smithi145 bash[21339]: audit 2024-03-07T03:00:01.220978+0000 mon.a (mon.0) 319 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:02.547 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:02 smithi145 bash[21339]: audit 2024-03-07T03:00:01.222216+0000 mon.a (mon.0) 320 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:02.547 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:02 smithi145 bash[21339]: audit 2024-03-07T03:00:01.223367+0000 mon.a (mon.0) 321 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:02.547 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:02 smithi145 bash[21339]: audit 2024-03-07T03:00:01.229505+0000 mon.a (mon.0) 322 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:02.547 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:02 smithi145 bash[30906]: audit 2024-03-07T03:00:01.215093+0000 mon.a (mon.0) 318 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:02.547 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:02 smithi145 bash[30906]: audit 2024-03-07T03:00:01.220978+0000 mon.a (mon.0) 319 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:02.547 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:02 smithi145 bash[30906]: audit 2024-03-07T03:00:01.222216+0000 mon.a (mon.0) 320 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:02.547 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:02 smithi145 bash[30906]: audit 2024-03-07T03:00:01.223367+0000 mon.a (mon.0) 321 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:02.547 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:02 smithi145 bash[30906]: audit 2024-03-07T03:00:01.229505+0000 mon.a (mon.0) 322 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:02.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:02 smithi189 bash[23614]: audit 2024-03-07T03:00:01.215093+0000 mon.a (mon.0) 318 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:02.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:02 smithi189 bash[23614]: audit 2024-03-07T03:00:01.220978+0000 mon.a (mon.0) 319 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:02.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:02 smithi189 bash[23614]: audit 2024-03-07T03:00:01.222216+0000 mon.a (mon.0) 320 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:02.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:02 smithi189 bash[23614]: audit 2024-03-07T03:00:01.223367+0000 mon.a (mon.0) 321 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:02.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:02 smithi189 bash[23614]: audit 2024-03-07T03:00:01.229505+0000 mon.a (mon.0) 322 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:03.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:03 smithi145 bash[21339]: cluster 2024-03-07T03:00:01.260484+0000 mgr.y (mgr.14152) 134 : cluster 0 pgmap v97: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:00:03.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:03 smithi145 bash[21339]: audit 2024-03-07T03:00:02.552000+0000 mon.a (mon.0) 323 : audit 1 from='osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:00:03.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:03 smithi145 bash[30906]: cluster 2024-03-07T03:00:01.260484+0000 mgr.y (mgr.14152) 134 : cluster 0 pgmap v97: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:00:03.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:03 smithi145 bash[30906]: audit 2024-03-07T03:00:02.552000+0000 mon.a (mon.0) 323 : audit 1 from='osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:00:03.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:03 smithi189 bash[23614]: cluster 2024-03-07T03:00:01.260484+0000 mgr.y (mgr.14152) 134 : cluster 0 pgmap v97: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:00:03.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:03 smithi189 bash[23614]: audit 2024-03-07T03:00:02.552000+0000 mon.a (mon.0) 323 : audit 1 from='osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:00:04.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:04 smithi145 bash[30906]: audit 2024-03-07T03:00:03.246875+0000 mon.a (mon.0) 324 : audit 1 from='osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-07T03:00:04.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:04 smithi145 bash[30906]: cluster 2024-03-07T03:00:03.247081+0000 mon.a (mon.0) 325 : cluster 0 osdmap e11: 2 total, 1 up, 2 in 2024-03-07T03:00:04.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:04 smithi145 bash[30906]: audit 2024-03-07T03:00:03.247333+0000 mon.a (mon.0) 326 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:04.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:04 smithi145 bash[30906]: audit 2024-03-07T03:00:03.248298+0000 mon.a (mon.0) 327 : audit 1 from='osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:00:04.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:04 smithi145 bash[21339]: audit 2024-03-07T03:00:03.246875+0000 mon.a (mon.0) 324 : audit 1 from='osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-07T03:00:04.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:04 smithi145 bash[21339]: cluster 2024-03-07T03:00:03.247081+0000 mon.a (mon.0) 325 : cluster 0 osdmap e11: 2 total, 1 up, 2 in 2024-03-07T03:00:04.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:04 smithi145 bash[21339]: audit 2024-03-07T03:00:03.247333+0000 mon.a (mon.0) 326 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:04.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:04 smithi145 bash[21339]: audit 2024-03-07T03:00:03.248298+0000 mon.a (mon.0) 327 : audit 1 from='osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:00:04.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:04 smithi189 bash[23614]: audit 2024-03-07T03:00:03.246875+0000 mon.a (mon.0) 324 : audit 1 from='osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-07T03:00:04.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:04 smithi189 bash[23614]: cluster 2024-03-07T03:00:03.247081+0000 mon.a (mon.0) 325 : cluster 0 osdmap e11: 2 total, 1 up, 2 in 2024-03-07T03:00:04.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:04 smithi189 bash[23614]: audit 2024-03-07T03:00:03.247333+0000 mon.a (mon.0) 326 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:04.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:04 smithi189 bash[23614]: audit 2024-03-07T03:00:03.248298+0000 mon.a (mon.0) 327 : audit 1 from='osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:00:05.056 INFO:teuthology.orchestra.run.smithi145.stdout:Created osd(s) 1 on host 'smithi145' 2024-03-07T03:00:05.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[21339]: cluster 2024-03-07T03:00:03.260946+0000 mgr.y (mgr.14152) 135 : cluster 0 pgmap v99: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:00:05.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[21339]: audit 2024-03-07T03:00:04.251221+0000 mon.a (mon.0) 328 : audit 1 from='osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-03-07T03:00:05.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[21339]: cluster 2024-03-07T03:00:04.251293+0000 mon.a (mon.0) 329 : cluster 0 osdmap e12: 2 total, 1 up, 2 in 2024-03-07T03:00:05.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[21339]: audit 2024-03-07T03:00:04.251683+0000 mon.a (mon.0) 330 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:05.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[21339]: audit 2024-03-07T03:00:04.256047+0000 mon.a (mon.0) 331 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:05.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[21339]: audit 2024-03-07T03:00:05.006231+0000 mon.a (mon.0) 332 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:05.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[21339]: audit 2024-03-07T03:00:05.007305+0000 mon.a (mon.0) 333 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:05.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[21339]: audit 2024-03-07T03:00:05.016307+0000 mon.a (mon.0) 334 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:05.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[30906]: cluster 2024-03-07T03:00:03.260946+0000 mgr.y (mgr.14152) 135 : cluster 0 pgmap v99: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:00:05.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[30906]: audit 2024-03-07T03:00:04.251221+0000 mon.a (mon.0) 328 : audit 1 from='osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-03-07T03:00:05.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[30906]: cluster 2024-03-07T03:00:04.251293+0000 mon.a (mon.0) 329 : cluster 0 osdmap e12: 2 total, 1 up, 2 in 2024-03-07T03:00:05.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[30906]: audit 2024-03-07T03:00:04.251683+0000 mon.a (mon.0) 330 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:05.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[30906]: audit 2024-03-07T03:00:04.256047+0000 mon.a (mon.0) 331 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:05.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[30906]: audit 2024-03-07T03:00:05.006231+0000 mon.a (mon.0) 332 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:05.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[30906]: audit 2024-03-07T03:00:05.007305+0000 mon.a (mon.0) 333 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:05.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:05 smithi145 bash[30906]: audit 2024-03-07T03:00:05.016307+0000 mon.a (mon.0) 334 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:05.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:05 smithi189 bash[23614]: cluster 2024-03-07T03:00:03.260946+0000 mgr.y (mgr.14152) 135 : cluster 0 pgmap v99: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:00:05.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:05 smithi189 bash[23614]: audit 2024-03-07T03:00:04.251221+0000 mon.a (mon.0) 328 : audit 1 from='osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-03-07T03:00:05.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:05 smithi189 bash[23614]: cluster 2024-03-07T03:00:04.251293+0000 mon.a (mon.0) 329 : cluster 0 osdmap e12: 2 total, 1 up, 2 in 2024-03-07T03:00:05.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:05 smithi189 bash[23614]: audit 2024-03-07T03:00:04.251683+0000 mon.a (mon.0) 330 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:05.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:05 smithi189 bash[23614]: audit 2024-03-07T03:00:04.256047+0000 mon.a (mon.0) 331 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:05.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:05 smithi189 bash[23614]: audit 2024-03-07T03:00:05.006231+0000 mon.a (mon.0) 332 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:05.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:05 smithi189 bash[23614]: audit 2024-03-07T03:00:05.007305+0000 mon.a (mon.0) 333 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:05.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:05 smithi189 bash[23614]: audit 2024-03-07T03:00:05.016307+0000 mon.a (mon.0) 334 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:05.729 DEBUG:teuthology.orchestra.run.smithi145:osd.1> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.1.service 2024-03-07T03:00:05.732 INFO:tasks.cephadm:Deploying osd.2 on smithi145 with /dev/vg_nvme/lv_2... 2024-03-07T03:00:05.732 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-07T03:00:06.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[21339]: cluster 2024-03-07T03:00:03.501582+0000 osd.1 (osd.1) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:00:06.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[21339]: cluster 2024-03-07T03:00:03.501742+0000 osd.1 (osd.1) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:00:06.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[21339]: cluster 2024-03-07T03:00:05.253930+0000 mon.a (mon.0) 335 : cluster 1 osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359] boot 2024-03-07T03:00:06.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[21339]: cluster 2024-03-07T03:00:05.254024+0000 mon.a (mon.0) 336 : cluster 0 osdmap e13: 2 total, 2 up, 2 in 2024-03-07T03:00:06.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[21339]: audit 2024-03-07T03:00:05.255420+0000 mon.a (mon.0) 337 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:06.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[21339]: cluster 2024-03-07T03:00:05.261356+0000 mgr.y (mgr.14152) 136 : cluster 0 pgmap v102: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:00:06.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[21339]: audit 2024-03-07T03:00:05.264935+0000 mon.a (mon.0) 338 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:06.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[30906]: cluster 2024-03-07T03:00:03.501582+0000 osd.1 (osd.1) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:00:06.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[30906]: cluster 2024-03-07T03:00:03.501742+0000 osd.1 (osd.1) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:00:06.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[30906]: cluster 2024-03-07T03:00:05.253930+0000 mon.a (mon.0) 335 : cluster 1 osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359] boot 2024-03-07T03:00:06.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[30906]: cluster 2024-03-07T03:00:05.254024+0000 mon.a (mon.0) 336 : cluster 0 osdmap e13: 2 total, 2 up, 2 in 2024-03-07T03:00:06.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[30906]: audit 2024-03-07T03:00:05.255420+0000 mon.a (mon.0) 337 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:06.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[30906]: cluster 2024-03-07T03:00:05.261356+0000 mgr.y (mgr.14152) 136 : cluster 0 pgmap v102: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:00:06.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:06 smithi145 bash[30906]: audit 2024-03-07T03:00:05.264935+0000 mon.a (mon.0) 338 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:06.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:06 smithi189 bash[23614]: cluster 2024-03-07T03:00:03.501582+0000 osd.1 (osd.1) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:00:06.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:06 smithi189 bash[23614]: cluster 2024-03-07T03:00:03.501742+0000 osd.1 (osd.1) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:00:06.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:06 smithi189 bash[23614]: cluster 2024-03-07T03:00:05.253930+0000 mon.a (mon.0) 335 : cluster 1 osd.1 [v2:172.21.15.145:6810/1873038359,v1:172.21.15.145:6811/1873038359] boot 2024-03-07T03:00:06.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:06 smithi189 bash[23614]: cluster 2024-03-07T03:00:05.254024+0000 mon.a (mon.0) 336 : cluster 0 osdmap e13: 2 total, 2 up, 2 in 2024-03-07T03:00:06.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:06 smithi189 bash[23614]: audit 2024-03-07T03:00:05.255420+0000 mon.a (mon.0) 337 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:06.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:06 smithi189 bash[23614]: cluster 2024-03-07T03:00:05.261356+0000 mgr.y (mgr.14152) 136 : cluster 0 pgmap v102: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:00:06.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:06 smithi189 bash[23614]: audit 2024-03-07T03:00:05.264935+0000 mon.a (mon.0) 338 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:00:07.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:07 smithi145 bash[21339]: cluster 2024-03-07T03:00:06.272060+0000 mon.a (mon.0) 339 : cluster 0 osdmap e14: 2 total, 2 up, 2 in 2024-03-07T03:00:07.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:07 smithi145 bash[30906]: cluster 2024-03-07T03:00:06.272060+0000 mon.a (mon.0) 339 : cluster 0 osdmap e14: 2 total, 2 up, 2 in 2024-03-07T03:00:07.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:07 smithi189 bash[23614]: cluster 2024-03-07T03:00:06.272060+0000 mon.a (mon.0) 339 : cluster 0 osdmap e14: 2 total, 2 up, 2 in 2024-03-07T03:00:09.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:09 smithi145 bash[21339]: cluster 2024-03-07T03:00:07.261793+0000 mgr.y (mgr.14152) 137 : cluster 0 pgmap v104: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:09.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:09 smithi145 bash[30906]: cluster 2024-03-07T03:00:07.261793+0000 mgr.y (mgr.14152) 137 : cluster 0 pgmap v104: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:09.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:09 smithi189 bash[23614]: cluster 2024-03-07T03:00:07.261793+0000 mgr.y (mgr.14152) 137 : cluster 0 pgmap v104: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:09.878 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:00:11.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:11 smithi145 bash[21339]: cluster 2024-03-07T03:00:09.262298+0000 mgr.y (mgr.14152) 138 : cluster 0 pgmap v105: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:11.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:11 smithi145 bash[30906]: cluster 2024-03-07T03:00:09.262298+0000 mgr.y (mgr.14152) 138 : cluster 0 pgmap v105: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:11.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:11 smithi189 bash[23614]: cluster 2024-03-07T03:00:09.262298+0000 mgr.y (mgr.14152) 138 : cluster 0 pgmap v105: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:13.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:13 smithi145 bash[21339]: cluster 2024-03-07T03:00:11.262583+0000 mgr.y (mgr.14152) 139 : cluster 0 pgmap v106: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:13.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:13 smithi145 bash[30906]: cluster 2024-03-07T03:00:11.262583+0000 mgr.y (mgr.14152) 139 : cluster 0 pgmap v106: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:13.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:13 smithi189 bash[23614]: cluster 2024-03-07T03:00:11.262583+0000 mgr.y (mgr.14152) 139 : cluster 0 pgmap v106: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:14.297 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-07T03:00:14.298 INFO:teuthology.orchestra.run.smithi145.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-07T03:00:14.298 INFO:teuthology.orchestra.run.smithi145.stderr: stderr: 10+0 records in 2024-03-07T03:00:14.298 INFO:teuthology.orchestra.run.smithi145.stderr:10+0 records out 2024-03-07T03:00:14.298 INFO:teuthology.orchestra.run.smithi145.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0136646 s, 767 MB/s 2024-03-07T03:00:14.298 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping successful for: 2024-03-07T03:00:15.222 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi145:vg_nvme/lv_2 2024-03-07T03:00:15.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:15 smithi145 bash[21339]: cluster 2024-03-07T03:00:13.262955+0000 mgr.y (mgr.14152) 140 : cluster 0 pgmap v107: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:15.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:15 smithi145 bash[30906]: cluster 2024-03-07T03:00:13.262955+0000 mgr.y (mgr.14152) 140 : cluster 0 pgmap v107: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:15.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:15 smithi189 bash[23614]: cluster 2024-03-07T03:00:13.262955+0000 mgr.y (mgr.14152) 140 : cluster 0 pgmap v107: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:17.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[21339]: cluster 2024-03-07T03:00:15.263429+0000 mgr.y (mgr.14152) 141 : cluster 0 pgmap v108: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:17.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[21339]: cephadm 2024-03-07T03:00:15.601402+0000 mgr.y (mgr.14152) 142 : cephadm 1 Detected new or changed devices on smithi145 2024-03-07T03:00:17.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[21339]: audit 2024-03-07T03:00:15.611149+0000 mon.a (mon.0) 340 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:17.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[21339]: audit 2024-03-07T03:00:15.621084+0000 mon.a (mon.0) 341 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:17.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[21339]: audit 2024-03-07T03:00:15.623481+0000 mon.a (mon.0) 342 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:00:17.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[21339]: audit 2024-03-07T03:00:15.625367+0000 mon.a (mon.0) 343 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:00:17.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[21339]: cephadm 2024-03-07T03:00:15.626783+0000 mgr.y (mgr.14152) 143 : cephadm 1 Adjusting osd_memory_target on smithi145 to 8120M 2024-03-07T03:00:17.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[21339]: audit 2024-03-07T03:00:15.635654+0000 mon.a (mon.0) 344 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:17.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[21339]: audit 2024-03-07T03:00:15.637754+0000 mon.a (mon.0) 345 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:17.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[21339]: audit 2024-03-07T03:00:15.639085+0000 mon.a (mon.0) 346 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:17.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[21339]: audit 2024-03-07T03:00:15.649626+0000 mon.a (mon.0) 347 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:17.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[30906]: cluster 2024-03-07T03:00:15.263429+0000 mgr.y (mgr.14152) 141 : cluster 0 pgmap v108: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:17.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[30906]: cephadm 2024-03-07T03:00:15.601402+0000 mgr.y (mgr.14152) 142 : cephadm 1 Detected new or changed devices on smithi145 2024-03-07T03:00:17.072 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[30906]: audit 2024-03-07T03:00:15.611149+0000 mon.a (mon.0) 340 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:17.072 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[30906]: audit 2024-03-07T03:00:15.621084+0000 mon.a (mon.0) 341 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:17.072 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[30906]: audit 2024-03-07T03:00:15.623481+0000 mon.a (mon.0) 342 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:00:17.072 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[30906]: audit 2024-03-07T03:00:15.625367+0000 mon.a (mon.0) 343 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:00:17.072 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[30906]: cephadm 2024-03-07T03:00:15.626783+0000 mgr.y (mgr.14152) 143 : cephadm 1 Adjusting osd_memory_target on smithi145 to 8120M 2024-03-07T03:00:17.072 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[30906]: audit 2024-03-07T03:00:15.635654+0000 mon.a (mon.0) 344 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:17.072 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[30906]: audit 2024-03-07T03:00:15.637754+0000 mon.a (mon.0) 345 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:17.072 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[30906]: audit 2024-03-07T03:00:15.639085+0000 mon.a (mon.0) 346 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:17.072 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:16 smithi145 bash[30906]: audit 2024-03-07T03:00:15.649626+0000 mon.a (mon.0) 347 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:17.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:16 smithi189 bash[23614]: cluster 2024-03-07T03:00:15.263429+0000 mgr.y (mgr.14152) 141 : cluster 0 pgmap v108: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:17.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:16 smithi189 bash[23614]: cephadm 2024-03-07T03:00:15.601402+0000 mgr.y (mgr.14152) 142 : cephadm 1 Detected new or changed devices on smithi145 2024-03-07T03:00:17.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:16 smithi189 bash[23614]: audit 2024-03-07T03:00:15.611149+0000 mon.a (mon.0) 340 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:17.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:16 smithi189 bash[23614]: audit 2024-03-07T03:00:15.621084+0000 mon.a (mon.0) 341 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:17.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:16 smithi189 bash[23614]: audit 2024-03-07T03:00:15.623481+0000 mon.a (mon.0) 342 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:00:17.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:16 smithi189 bash[23614]: audit 2024-03-07T03:00:15.625367+0000 mon.a (mon.0) 343 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:00:17.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:16 smithi189 bash[23614]: cephadm 2024-03-07T03:00:15.626783+0000 mgr.y (mgr.14152) 143 : cephadm 1 Adjusting osd_memory_target on smithi145 to 8120M 2024-03-07T03:00:17.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:16 smithi189 bash[23614]: audit 2024-03-07T03:00:15.635654+0000 mon.a (mon.0) 344 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:17.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:16 smithi189 bash[23614]: audit 2024-03-07T03:00:15.637754+0000 mon.a (mon.0) 345 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:17.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:16 smithi189 bash[23614]: audit 2024-03-07T03:00:15.639085+0000 mon.a (mon.0) 346 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:17.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:16 smithi189 bash[23614]: audit 2024-03-07T03:00:15.649626+0000 mon.a (mon.0) 347 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:19.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:19 smithi145 bash[21339]: cluster 2024-03-07T03:00:17.263899+0000 mgr.y (mgr.14152) 144 : cluster 0 pgmap v109: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:19.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:19 smithi145 bash[30906]: cluster 2024-03-07T03:00:17.263899+0000 mgr.y (mgr.14152) 144 : cluster 0 pgmap v109: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:19.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:19 smithi189 bash[23614]: cluster 2024-03-07T03:00:17.263899+0000 mgr.y (mgr.14152) 144 : cluster 0 pgmap v109: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:19.917 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:00:21.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:21 smithi145 bash[21339]: cluster 2024-03-07T03:00:19.264355+0000 mgr.y (mgr.14152) 145 : cluster 0 pgmap v110: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:21.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:21 smithi145 bash[30906]: cluster 2024-03-07T03:00:19.264355+0000 mgr.y (mgr.14152) 145 : cluster 0 pgmap v110: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:21.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:21 smithi189 bash[23614]: cluster 2024-03-07T03:00:19.264355+0000 mgr.y (mgr.14152) 145 : cluster 0 pgmap v110: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:23.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:23 smithi145 bash[21339]: cluster 2024-03-07T03:00:21.264834+0000 mgr.y (mgr.14152) 146 : cluster 0 pgmap v111: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:23.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:23 smithi145 bash[21339]: audit 2024-03-07T03:00:22.874525+0000 mon.a (mon.0) 348 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:00:23.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:23 smithi145 bash[21339]: audit 2024-03-07T03:00:22.877790+0000 mon.a (mon.0) 349 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:00:23.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:23 smithi145 bash[21339]: audit 2024-03-07T03:00:22.878844+0000 mon.a (mon.0) 350 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:23.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:23 smithi145 bash[30906]: cluster 2024-03-07T03:00:21.264834+0000 mgr.y (mgr.14152) 146 : cluster 0 pgmap v111: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:23.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:23 smithi145 bash[30906]: audit 2024-03-07T03:00:22.874525+0000 mon.a (mon.0) 348 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:00:23.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:23 smithi145 bash[30906]: audit 2024-03-07T03:00:22.877790+0000 mon.a (mon.0) 349 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:00:23.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:23 smithi145 bash[30906]: audit 2024-03-07T03:00:22.878844+0000 mon.a (mon.0) 350 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:23.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:23 smithi189 bash[23614]: cluster 2024-03-07T03:00:21.264834+0000 mgr.y (mgr.14152) 146 : cluster 0 pgmap v111: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:23.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:23 smithi189 bash[23614]: audit 2024-03-07T03:00:22.874525+0000 mon.a (mon.0) 348 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:00:23.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:23 smithi189 bash[23614]: audit 2024-03-07T03:00:22.877790+0000 mon.a (mon.0) 349 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:00:23.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:23 smithi189 bash[23614]: audit 2024-03-07T03:00:22.878844+0000 mon.a (mon.0) 350 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:24.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:24 smithi145 bash[21339]: audit 2024-03-07T03:00:22.867910+0000 mgr.y (mgr.14152) 147 : audit 0 from='client.24164 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:00:24.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:24 smithi145 bash[30906]: audit 2024-03-07T03:00:22.867910+0000 mgr.y (mgr.14152) 147 : audit 0 from='client.24164 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:00:24.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:24 smithi189 bash[23614]: audit 2024-03-07T03:00:22.867910+0000 mgr.y (mgr.14152) 147 : audit 0 from='client.24164 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:00:25.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:25 smithi145 bash[30906]: cluster 2024-03-07T03:00:23.265321+0000 mgr.y (mgr.14152) 148 : cluster 0 pgmap v112: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:25.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:25 smithi145 bash[21339]: cluster 2024-03-07T03:00:23.265321+0000 mgr.y (mgr.14152) 148 : cluster 0 pgmap v112: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:25.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:25 smithi189 bash[23614]: cluster 2024-03-07T03:00:23.265321+0000 mgr.y (mgr.14152) 148 : cluster 0 pgmap v112: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:27.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:27 smithi145 bash[30906]: cluster 2024-03-07T03:00:25.265778+0000 mgr.y (mgr.14152) 149 : cluster 0 pgmap v113: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:27.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:27 smithi145 bash[21339]: cluster 2024-03-07T03:00:25.265778+0000 mgr.y (mgr.14152) 149 : cluster 0 pgmap v113: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:27.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:27 smithi189 bash[23614]: cluster 2024-03-07T03:00:25.265778+0000 mgr.y (mgr.14152) 149 : cluster 0 pgmap v113: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:29.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:29 smithi145 bash[30906]: cluster 2024-03-07T03:00:27.266272+0000 mgr.y (mgr.14152) 150 : cluster 0 pgmap v114: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:29.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:29 smithi145 bash[21339]: cluster 2024-03-07T03:00:27.266272+0000 mgr.y (mgr.14152) 150 : cluster 0 pgmap v114: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:29.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:29 smithi189 bash[23614]: cluster 2024-03-07T03:00:27.266272+0000 mgr.y (mgr.14152) 150 : cluster 0 pgmap v114: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:31.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:31 smithi145 bash[21339]: cluster 2024-03-07T03:00:29.266747+0000 mgr.y (mgr.14152) 151 : cluster 0 pgmap v115: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:31.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:31 smithi145 bash[21339]: audit 2024-03-07T03:00:31.194680+0000 mon.a (mon.0) 351 : audit 1 from='client.? 172.21.15.145:0/3876296408' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "905ec219-dd9c-4f30-bdab-ed78d1405c11"}]: dispatch 2024-03-07T03:00:31.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:31 smithi145 bash[21339]: audit 2024-03-07T03:00:31.200137+0000 mon.a (mon.0) 352 : audit 1 from='client.? 172.21.15.145:0/3876296408' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "905ec219-dd9c-4f30-bdab-ed78d1405c11"}]': finished 2024-03-07T03:00:31.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:31 smithi145 bash[21339]: cluster 2024-03-07T03:00:31.200198+0000 mon.a (mon.0) 353 : cluster 0 osdmap e15: 3 total, 2 up, 3 in 2024-03-07T03:00:31.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:31 smithi145 bash[21339]: audit 2024-03-07T03:00:31.200298+0000 mon.a (mon.0) 354 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:31.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:31 smithi145 bash[30906]: cluster 2024-03-07T03:00:29.266747+0000 mgr.y (mgr.14152) 151 : cluster 0 pgmap v115: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:31.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:31 smithi145 bash[30906]: audit 2024-03-07T03:00:31.194680+0000 mon.a (mon.0) 351 : audit 1 from='client.? 172.21.15.145:0/3876296408' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "905ec219-dd9c-4f30-bdab-ed78d1405c11"}]: dispatch 2024-03-07T03:00:31.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:31 smithi145 bash[30906]: audit 2024-03-07T03:00:31.200137+0000 mon.a (mon.0) 352 : audit 1 from='client.? 172.21.15.145:0/3876296408' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "905ec219-dd9c-4f30-bdab-ed78d1405c11"}]': finished 2024-03-07T03:00:31.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:31 smithi145 bash[30906]: cluster 2024-03-07T03:00:31.200198+0000 mon.a (mon.0) 353 : cluster 0 osdmap e15: 3 total, 2 up, 3 in 2024-03-07T03:00:31.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:31 smithi145 bash[30906]: audit 2024-03-07T03:00:31.200298+0000 mon.a (mon.0) 354 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:31.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:31 smithi189 bash[23614]: cluster 2024-03-07T03:00:29.266747+0000 mgr.y (mgr.14152) 151 : cluster 0 pgmap v115: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:31.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:31 smithi189 bash[23614]: audit 2024-03-07T03:00:31.194680+0000 mon.a (mon.0) 351 : audit 1 from='client.? 172.21.15.145:0/3876296408' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "905ec219-dd9c-4f30-bdab-ed78d1405c11"}]: dispatch 2024-03-07T03:00:31.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:31 smithi189 bash[23614]: audit 2024-03-07T03:00:31.200137+0000 mon.a (mon.0) 352 : audit 1 from='client.? 172.21.15.145:0/3876296408' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "905ec219-dd9c-4f30-bdab-ed78d1405c11"}]': finished 2024-03-07T03:00:31.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:31 smithi189 bash[23614]: cluster 2024-03-07T03:00:31.200198+0000 mon.a (mon.0) 353 : cluster 0 osdmap e15: 3 total, 2 up, 3 in 2024-03-07T03:00:31.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:31 smithi189 bash[23614]: audit 2024-03-07T03:00:31.200298+0000 mon.a (mon.0) 354 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:32.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:32 smithi145 bash[30906]: audit 2024-03-07T03:00:31.920680+0000 mon.a (mon.0) 355 : audit 0 from='client.? 172.21.15.145:0/3952180325' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:00:32.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:32 smithi145 bash[21339]: audit 2024-03-07T03:00:31.920680+0000 mon.a (mon.0) 355 : audit 0 from='client.? 172.21.15.145:0/3952180325' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:00:32.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:32 smithi189 bash[23614]: audit 2024-03-07T03:00:31.920680+0000 mon.a (mon.0) 355 : audit 0 from='client.? 172.21.15.145:0/3952180325' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:00:33.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:33 smithi145 bash[21339]: cluster 2024-03-07T03:00:31.267211+0000 mgr.y (mgr.14152) 152 : cluster 0 pgmap v117: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:33.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:33 smithi145 bash[30906]: cluster 2024-03-07T03:00:31.267211+0000 mgr.y (mgr.14152) 152 : cluster 0 pgmap v117: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:33.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:33 smithi189 bash[23614]: cluster 2024-03-07T03:00:31.267211+0000 mgr.y (mgr.14152) 152 : cluster 0 pgmap v117: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:35.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:35 smithi145 bash[30906]: cluster 2024-03-07T03:00:33.267661+0000 mgr.y (mgr.14152) 153 : cluster 0 pgmap v118: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:35.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:35 smithi145 bash[21339]: cluster 2024-03-07T03:00:33.267661+0000 mgr.y (mgr.14152) 153 : cluster 0 pgmap v118: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:35.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:35 smithi189 bash[23614]: cluster 2024-03-07T03:00:33.267661+0000 mgr.y (mgr.14152) 153 : cluster 0 pgmap v118: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:37.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:37 smithi145 bash[21339]: cluster 2024-03-07T03:00:35.268011+0000 mgr.y (mgr.14152) 154 : cluster 0 pgmap v119: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:37.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:37 smithi145 bash[30906]: cluster 2024-03-07T03:00:35.268011+0000 mgr.y (mgr.14152) 154 : cluster 0 pgmap v119: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:37.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:37 smithi189 bash[23614]: cluster 2024-03-07T03:00:35.268011+0000 mgr.y (mgr.14152) 154 : cluster 0 pgmap v119: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:39.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:39 smithi145 bash[21339]: cluster 2024-03-07T03:00:37.268479+0000 mgr.y (mgr.14152) 155 : cluster 0 pgmap v120: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:39.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:39 smithi145 bash[30906]: cluster 2024-03-07T03:00:37.268479+0000 mgr.y (mgr.14152) 155 : cluster 0 pgmap v120: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:39.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:39 smithi189 bash[23614]: cluster 2024-03-07T03:00:37.268479+0000 mgr.y (mgr.14152) 155 : cluster 0 pgmap v120: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:41.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:41 smithi145 bash[21339]: cluster 2024-03-07T03:00:39.269020+0000 mgr.y (mgr.14152) 156 : cluster 0 pgmap v121: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:41 smithi145 bash[30906]: cluster 2024-03-07T03:00:39.269020+0000 mgr.y (mgr.14152) 156 : cluster 0 pgmap v121: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:41.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:41 smithi189 bash[23614]: cluster 2024-03-07T03:00:39.269020+0000 mgr.y (mgr.14152) 156 : cluster 0 pgmap v121: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:42.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:42 smithi145 bash[21339]: cluster 2024-03-07T03:00:41.269519+0000 mgr.y (mgr.14152) 157 : cluster 0 pgmap v122: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:42.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:42 smithi145 bash[30906]: cluster 2024-03-07T03:00:41.269519+0000 mgr.y (mgr.14152) 157 : cluster 0 pgmap v122: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:42.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:42 smithi189 bash[23614]: cluster 2024-03-07T03:00:41.269519+0000 mgr.y (mgr.14152) 157 : cluster 0 pgmap v122: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:44.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:44 smithi145 bash[21339]: audit 2024-03-07T03:00:43.871481+0000 mon.a (mon.0) 356 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:00:44.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:44 smithi145 bash[21339]: audit 2024-03-07T03:00:43.873201+0000 mon.a (mon.0) 357 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:44.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:44 smithi145 bash[30906]: audit 2024-03-07T03:00:43.871481+0000 mon.a (mon.0) 356 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:00:44.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:44 smithi145 bash[30906]: audit 2024-03-07T03:00:43.873201+0000 mon.a (mon.0) 357 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:44.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:44 smithi189 bash[23614]: audit 2024-03-07T03:00:43.871481+0000 mon.a (mon.0) 356 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:00:44.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:44 smithi189 bash[23614]: audit 2024-03-07T03:00:43.873201+0000 mon.a (mon.0) 357 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:45.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:45 smithi145 bash[21339]: cluster 2024-03-07T03:00:43.269972+0000 mgr.y (mgr.14152) 158 : cluster 0 pgmap v123: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:45.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:45 smithi145 bash[21339]: cephadm 2024-03-07T03:00:43.874680+0000 mgr.y (mgr.14152) 159 : cephadm 1 Deploying daemon osd.2 on smithi145 2024-03-07T03:00:45.457 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:45 smithi145 bash[30906]: cluster 2024-03-07T03:00:43.269972+0000 mgr.y (mgr.14152) 158 : cluster 0 pgmap v123: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:45.457 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:45 smithi145 bash[30906]: cephadm 2024-03-07T03:00:43.874680+0000 mgr.y (mgr.14152) 159 : cephadm 1 Deploying daemon osd.2 on smithi145 2024-03-07T03:00:45.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:45 smithi189 bash[23614]: cluster 2024-03-07T03:00:43.269972+0000 mgr.y (mgr.14152) 158 : cluster 0 pgmap v123: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:45.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:45 smithi189 bash[23614]: cephadm 2024-03-07T03:00:43.874680+0000 mgr.y (mgr.14152) 159 : cephadm 1 Deploying daemon osd.2 on smithi145 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:47 smithi145 bash[21339]: cluster 2024-03-07T03:00:45.270323+0000 mgr.y (mgr.14152) 160 : cluster 0 pgmap v124: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:47 smithi145 bash[30906]: cluster 2024-03-07T03:00:45.270323+0000 mgr.y (mgr.14152) 160 : cluster 0 pgmap v124: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:47.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:47 smithi189 bash[23614]: cluster 2024-03-07T03:00:45.270323+0000 mgr.y (mgr.14152) 160 : cluster 0 pgmap v124: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:48.169 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:48 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:48.170 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:48 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:48.170 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:00:48 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:48.170 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:00:48 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:48.170 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:00:48 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:48.433 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:48 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:48.433 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:00:48 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:48.433 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:00:48 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:48.434 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:48 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:48.434 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:00:48 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:49.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:49 smithi145 bash[21339]: cluster 2024-03-07T03:00:47.270726+0000 mgr.y (mgr.14152) 161 : cluster 0 pgmap v125: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:49.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:49 smithi145 bash[21339]: audit 2024-03-07T03:00:48.473123+0000 mon.a (mon.0) 358 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:49.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:49 smithi145 bash[21339]: audit 2024-03-07T03:00:48.478347+0000 mon.a (mon.0) 359 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:49.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:49 smithi145 bash[21339]: audit 2024-03-07T03:00:48.515674+0000 mon.a (mon.0) 360 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:49.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:49 smithi145 bash[30906]: cluster 2024-03-07T03:00:47.270726+0000 mgr.y (mgr.14152) 161 : cluster 0 pgmap v125: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:49.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:49 smithi145 bash[30906]: audit 2024-03-07T03:00:48.473123+0000 mon.a (mon.0) 358 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:49.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:49 smithi145 bash[30906]: audit 2024-03-07T03:00:48.478347+0000 mon.a (mon.0) 359 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:49.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:49 smithi145 bash[30906]: audit 2024-03-07T03:00:48.515674+0000 mon.a (mon.0) 360 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:49 smithi189 bash[23614]: cluster 2024-03-07T03:00:47.270726+0000 mgr.y (mgr.14152) 161 : cluster 0 pgmap v125: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:49 smithi189 bash[23614]: audit 2024-03-07T03:00:48.473123+0000 mon.a (mon.0) 358 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:49 smithi189 bash[23614]: audit 2024-03-07T03:00:48.478347+0000 mon.a (mon.0) 359 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:49 smithi189 bash[23614]: audit 2024-03-07T03:00:48.515674+0000 mon.a (mon.0) 360 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:51.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:51 smithi145 bash[30906]: cluster 2024-03-07T03:00:49.271169+0000 mgr.y (mgr.14152) 162 : cluster 0 pgmap v126: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:51.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:51 smithi145 bash[21339]: cluster 2024-03-07T03:00:49.271169+0000 mgr.y (mgr.14152) 162 : cluster 0 pgmap v126: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:51.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:51 smithi189 bash[23614]: cluster 2024-03-07T03:00:49.271169+0000 mgr.y (mgr.14152) 162 : cluster 0 pgmap v126: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:53.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:53 smithi145 bash[21339]: cluster 2024-03-07T03:00:51.271652+0000 mgr.y (mgr.14152) 163 : cluster 0 pgmap v127: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:53.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:53 smithi145 bash[30906]: cluster 2024-03-07T03:00:51.271652+0000 mgr.y (mgr.14152) 163 : cluster 0 pgmap v127: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:53.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:53 smithi189 bash[23614]: cluster 2024-03-07T03:00:51.271652+0000 mgr.y (mgr.14152) 163 : cluster 0 pgmap v127: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:55.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:55 smithi145 bash[30906]: cluster 2024-03-07T03:00:53.272120+0000 mgr.y (mgr.14152) 164 : cluster 0 pgmap v128: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:55.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:55 smithi145 bash[30906]: audit 2024-03-07T03:00:54.048924+0000 mon.a (mon.0) 361 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:55.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:55 smithi145 bash[30906]: audit 2024-03-07T03:00:54.054017+0000 mon.a (mon.0) 362 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:55.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:55 smithi145 bash[30906]: audit 2024-03-07T03:00:54.525384+0000 mon.a (mon.0) 363 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:55.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:55 smithi145 bash[30906]: audit 2024-03-07T03:00:54.527112+0000 mon.a (mon.0) 364 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:55.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:55 smithi145 bash[30906]: audit 2024-03-07T03:00:54.536847+0000 mon.a (mon.0) 365 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:55.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:55 smithi145 bash[21339]: cluster 2024-03-07T03:00:53.272120+0000 mgr.y (mgr.14152) 164 : cluster 0 pgmap v128: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:55.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:55 smithi145 bash[21339]: audit 2024-03-07T03:00:54.048924+0000 mon.a (mon.0) 361 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:55.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:55 smithi145 bash[21339]: audit 2024-03-07T03:00:54.054017+0000 mon.a (mon.0) 362 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:55.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:55 smithi145 bash[21339]: audit 2024-03-07T03:00:54.525384+0000 mon.a (mon.0) 363 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:55.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:55 smithi145 bash[21339]: audit 2024-03-07T03:00:54.527112+0000 mon.a (mon.0) 364 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:55.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:55 smithi145 bash[21339]: audit 2024-03-07T03:00:54.536847+0000 mon.a (mon.0) 365 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:55.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:55 smithi189 bash[23614]: cluster 2024-03-07T03:00:53.272120+0000 mgr.y (mgr.14152) 164 : cluster 0 pgmap v128: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:55.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:55 smithi189 bash[23614]: audit 2024-03-07T03:00:54.048924+0000 mon.a (mon.0) 361 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:55.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:55 smithi189 bash[23614]: audit 2024-03-07T03:00:54.054017+0000 mon.a (mon.0) 362 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:55.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:55 smithi189 bash[23614]: audit 2024-03-07T03:00:54.525384+0000 mon.a (mon.0) 363 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:55.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:55 smithi189 bash[23614]: audit 2024-03-07T03:00:54.527112+0000 mon.a (mon.0) 364 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:55.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:55 smithi189 bash[23614]: audit 2024-03-07T03:00:54.536847+0000 mon.a (mon.0) 365 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:56.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:56 smithi145 bash[21339]: audit 2024-03-07T03:00:55.629348+0000 mon.a (mon.0) 366 : audit 1 from='osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:00:56.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:56 smithi145 bash[30906]: audit 2024-03-07T03:00:55.629348+0000 mon.a (mon.0) 366 : audit 1 from='osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:00:56.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:56 smithi189 bash[23614]: audit 2024-03-07T03:00:55.629348+0000 mon.a (mon.0) 366 : audit 1 from='osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:00:57.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:57 smithi145 bash[30906]: cluster 2024-03-07T03:00:55.272582+0000 mgr.y (mgr.14152) 165 : cluster 0 pgmap v129: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:57.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:57 smithi145 bash[30906]: audit 2024-03-07T03:00:56.062953+0000 mon.a (mon.0) 367 : audit 1 from='osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-07T03:00:57.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:57 smithi145 bash[30906]: cluster 2024-03-07T03:00:56.063175+0000 mon.a (mon.0) 368 : cluster 0 osdmap e16: 3 total, 2 up, 3 in 2024-03-07T03:00:57.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:57 smithi145 bash[30906]: audit 2024-03-07T03:00:56.063411+0000 mon.a (mon.0) 369 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:57.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:57 smithi145 bash[30906]: audit 2024-03-07T03:00:56.064289+0000 mon.a (mon.0) 370 : audit 1 from='osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:00:57.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:57 smithi145 bash[21339]: cluster 2024-03-07T03:00:55.272582+0000 mgr.y (mgr.14152) 165 : cluster 0 pgmap v129: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:57.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:57 smithi145 bash[21339]: audit 2024-03-07T03:00:56.062953+0000 mon.a (mon.0) 367 : audit 1 from='osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-07T03:00:57.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:57 smithi145 bash[21339]: cluster 2024-03-07T03:00:56.063175+0000 mon.a (mon.0) 368 : cluster 0 osdmap e16: 3 total, 2 up, 3 in 2024-03-07T03:00:57.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:57 smithi145 bash[21339]: audit 2024-03-07T03:00:56.063411+0000 mon.a (mon.0) 369 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:57.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:57 smithi145 bash[21339]: audit 2024-03-07T03:00:56.064289+0000 mon.a (mon.0) 370 : audit 1 from='osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:00:57.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:57 smithi189 bash[23614]: cluster 2024-03-07T03:00:55.272582+0000 mgr.y (mgr.14152) 165 : cluster 0 pgmap v129: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:57.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:57 smithi189 bash[23614]: audit 2024-03-07T03:00:56.062953+0000 mon.a (mon.0) 367 : audit 1 from='osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-07T03:00:57.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:57 smithi189 bash[23614]: cluster 2024-03-07T03:00:56.063175+0000 mon.a (mon.0) 368 : cluster 0 osdmap e16: 3 total, 2 up, 3 in 2024-03-07T03:00:57.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:57 smithi189 bash[23614]: audit 2024-03-07T03:00:56.063411+0000 mon.a (mon.0) 369 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:57.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:57 smithi189 bash[23614]: audit 2024-03-07T03:00:56.064289+0000 mon.a (mon.0) 370 : audit 1 from='osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:00:57.932 INFO:teuthology.orchestra.run.smithi145.stdout:Created osd(s) 2 on host 'smithi145' 2024-03-07T03:00:58.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[30906]: audit 2024-03-07T03:00:57.068105+0000 mon.a (mon.0) 371 : audit 1 from='osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-03-07T03:00:58.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[30906]: cluster 2024-03-07T03:00:57.068273+0000 mon.a (mon.0) 372 : cluster 0 osdmap e17: 3 total, 2 up, 3 in 2024-03-07T03:00:58.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[30906]: audit 2024-03-07T03:00:57.068847+0000 mon.a (mon.0) 373 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:58.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[30906]: audit 2024-03-07T03:00:57.076493+0000 mon.a (mon.0) 374 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:58.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[30906]: audit 2024-03-07T03:00:57.918825+0000 mon.a (mon.0) 375 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:58.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[30906]: audit 2024-03-07T03:00:57.930338+0000 mon.a (mon.0) 376 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:58.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[30906]: audit 2024-03-07T03:00:57.930962+0000 mon.a (mon.0) 377 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:58.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[21339]: audit 2024-03-07T03:00:57.068105+0000 mon.a (mon.0) 371 : audit 1 from='osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-03-07T03:00:58.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[21339]: cluster 2024-03-07T03:00:57.068273+0000 mon.a (mon.0) 372 : cluster 0 osdmap e17: 3 total, 2 up, 3 in 2024-03-07T03:00:58.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[21339]: audit 2024-03-07T03:00:57.068847+0000 mon.a (mon.0) 373 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:58.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[21339]: audit 2024-03-07T03:00:57.076493+0000 mon.a (mon.0) 374 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:58.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[21339]: audit 2024-03-07T03:00:57.918825+0000 mon.a (mon.0) 375 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:58.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[21339]: audit 2024-03-07T03:00:57.930338+0000 mon.a (mon.0) 376 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:58.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:58 smithi145 bash[21339]: audit 2024-03-07T03:00:57.930962+0000 mon.a (mon.0) 377 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:58.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:58 smithi189 bash[23614]: audit 2024-03-07T03:00:57.068105+0000 mon.a (mon.0) 371 : audit 1 from='osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-03-07T03:00:58.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:58 smithi189 bash[23614]: cluster 2024-03-07T03:00:57.068273+0000 mon.a (mon.0) 372 : cluster 0 osdmap e17: 3 total, 2 up, 3 in 2024-03-07T03:00:58.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:58 smithi189 bash[23614]: audit 2024-03-07T03:00:57.068847+0000 mon.a (mon.0) 373 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:58.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:58 smithi189 bash[23614]: audit 2024-03-07T03:00:57.076493+0000 mon.a (mon.0) 374 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:58.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:58 smithi189 bash[23614]: audit 2024-03-07T03:00:57.918825+0000 mon.a (mon.0) 375 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:58.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:58 smithi189 bash[23614]: audit 2024-03-07T03:00:57.930338+0000 mon.a (mon.0) 376 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:00:58.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:58 smithi189 bash[23614]: audit 2024-03-07T03:00:57.930962+0000 mon.a (mon.0) 377 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:58.664 DEBUG:teuthology.orchestra.run.smithi145:osd.2> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.2.service 2024-03-07T03:00:58.667 INFO:tasks.cephadm:Deploying osd.3 on smithi145 with /dev/vg_nvme/lv_1... 2024-03-07T03:00:58.667 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-07T03:00:59.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:59 smithi189 bash[23614]: cluster 2024-03-07T03:00:56.634933+0000 osd.2 (osd.2) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:00:59.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:59 smithi189 bash[23614]: cluster 2024-03-07T03:00:56.635001+0000 osd.2 (osd.2) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:00:59.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:59 smithi189 bash[23614]: cluster 2024-03-07T03:00:57.273154+0000 mgr.y (mgr.14152) 166 : cluster 0 pgmap v132: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:59.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:59 smithi189 bash[23614]: audit 2024-03-07T03:00:58.073370+0000 mon.a (mon.0) 378 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:59.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:59 smithi189 bash[23614]: cluster 2024-03-07T03:00:58.079094+0000 mon.a (mon.0) 379 : cluster 1 osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015] boot 2024-03-07T03:00:59.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:59 smithi189 bash[23614]: cluster 2024-03-07T03:00:58.079135+0000 mon.a (mon.0) 380 : cluster 0 osdmap e18: 3 total, 3 up, 3 in 2024-03-07T03:00:59.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:00:59 smithi189 bash[23614]: audit 2024-03-07T03:00:58.080126+0000 mon.a (mon.0) 381 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:59.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[21339]: cluster 2024-03-07T03:00:56.634933+0000 osd.2 (osd.2) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:00:59.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[21339]: cluster 2024-03-07T03:00:56.635001+0000 osd.2 (osd.2) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:00:59.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[21339]: cluster 2024-03-07T03:00:57.273154+0000 mgr.y (mgr.14152) 166 : cluster 0 pgmap v132: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:59.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[21339]: audit 2024-03-07T03:00:58.073370+0000 mon.a (mon.0) 378 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:59.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[21339]: cluster 2024-03-07T03:00:58.079094+0000 mon.a (mon.0) 379 : cluster 1 osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015] boot 2024-03-07T03:00:59.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[21339]: cluster 2024-03-07T03:00:58.079135+0000 mon.a (mon.0) 380 : cluster 0 osdmap e18: 3 total, 3 up, 3 in 2024-03-07T03:00:59.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[21339]: audit 2024-03-07T03:00:58.080126+0000 mon.a (mon.0) 381 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:59.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[30906]: cluster 2024-03-07T03:00:56.634933+0000 osd.2 (osd.2) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:00:59.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[30906]: cluster 2024-03-07T03:00:56.635001+0000 osd.2 (osd.2) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:00:59.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[30906]: cluster 2024-03-07T03:00:57.273154+0000 mgr.y (mgr.14152) 166 : cluster 0 pgmap v132: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:00:59.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[30906]: audit 2024-03-07T03:00:58.073370+0000 mon.a (mon.0) 378 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:00:59.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[30906]: cluster 2024-03-07T03:00:58.079094+0000 mon.a (mon.0) 379 : cluster 1 osd.2 [v2:172.21.15.145:6818/462216015,v1:172.21.15.145:6819/462216015] boot 2024-03-07T03:00:59.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[30906]: cluster 2024-03-07T03:00:58.079135+0000 mon.a (mon.0) 380 : cluster 0 osdmap e18: 3 total, 3 up, 3 in 2024-03-07T03:00:59.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:00:59 smithi145 bash[30906]: audit 2024-03-07T03:00:58.080126+0000 mon.a (mon.0) 381 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:01:00.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:00 smithi145 bash[21339]: cluster 2024-03-07T03:00:59.219332+0000 mon.a (mon.0) 382 : cluster 0 osdmap e19: 3 total, 3 up, 3 in 2024-03-07T03:01:00.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:00 smithi145 bash[21339]: audit 2024-03-07T03:00:59.401740+0000 mon.a (mon.0) 383 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:01:00.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:00 smithi145 bash[30906]: cluster 2024-03-07T03:00:59.219332+0000 mon.a (mon.0) 382 : cluster 0 osdmap e19: 3 total, 3 up, 3 in 2024-03-07T03:01:00.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:00 smithi145 bash[30906]: audit 2024-03-07T03:00:59.401740+0000 mon.a (mon.0) 383 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:01:00.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:00 smithi189 bash[23614]: cluster 2024-03-07T03:00:59.219332+0000 mon.a (mon.0) 382 : cluster 0 osdmap e19: 3 total, 3 up, 3 in 2024-03-07T03:01:00.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:00 smithi189 bash[23614]: audit 2024-03-07T03:00:59.401740+0000 mon.a (mon.0) 383 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:01:01.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:01 smithi145 bash[30906]: cluster 2024-03-07T03:00:59.273672+0000 mgr.y (mgr.14152) 167 : cluster 0 pgmap v135: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:01.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:01 smithi145 bash[30906]: audit 2024-03-07T03:01:00.228835+0000 mon.a (mon.0) 384 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-03-07T03:01:01.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:01 smithi145 bash[30906]: cluster 2024-03-07T03:01:00.229046+0000 mon.a (mon.0) 385 : cluster 0 osdmap e20: 3 total, 3 up, 3 in 2024-03-07T03:01:01.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:01 smithi145 bash[30906]: audit 2024-03-07T03:01:00.231022+0000 mon.a (mon.0) 386 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:01:01.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:01 smithi145 bash[21339]: cluster 2024-03-07T03:00:59.273672+0000 mgr.y (mgr.14152) 167 : cluster 0 pgmap v135: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:01.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:01 smithi145 bash[21339]: audit 2024-03-07T03:01:00.228835+0000 mon.a (mon.0) 384 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-03-07T03:01:01.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:01 smithi145 bash[21339]: cluster 2024-03-07T03:01:00.229046+0000 mon.a (mon.0) 385 : cluster 0 osdmap e20: 3 total, 3 up, 3 in 2024-03-07T03:01:01.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:01 smithi145 bash[21339]: audit 2024-03-07T03:01:00.231022+0000 mon.a (mon.0) 386 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:01:01.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:01 smithi189 bash[23614]: cluster 2024-03-07T03:00:59.273672+0000 mgr.y (mgr.14152) 167 : cluster 0 pgmap v135: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:01.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:01 smithi189 bash[23614]: audit 2024-03-07T03:01:00.228835+0000 mon.a (mon.0) 384 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-03-07T03:01:01.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:01 smithi189 bash[23614]: cluster 2024-03-07T03:01:00.229046+0000 mon.a (mon.0) 385 : cluster 0 osdmap e20: 3 total, 3 up, 3 in 2024-03-07T03:01:01.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:01 smithi189 bash[23614]: audit 2024-03-07T03:01:00.231022+0000 mon.a (mon.0) 386 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:01:02.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:02 smithi145 bash[21339]: audit 2024-03-07T03:01:01.229622+0000 mon.a (mon.0) 387 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-07T03:01:02.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:02 smithi145 bash[21339]: cluster 2024-03-07T03:01:01.229866+0000 mon.a (mon.0) 388 : cluster 0 osdmap e21: 3 total, 3 up, 3 in 2024-03-07T03:01:02.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:02 smithi145 bash[30906]: audit 2024-03-07T03:01:01.229622+0000 mon.a (mon.0) 387 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-07T03:01:02.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:02 smithi145 bash[30906]: cluster 2024-03-07T03:01:01.229866+0000 mon.a (mon.0) 388 : cluster 0 osdmap e21: 3 total, 3 up, 3 in 2024-03-07T03:01:02.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:02 smithi189 bash[23614]: audit 2024-03-07T03:01:01.229622+0000 mon.a (mon.0) 387 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-07T03:01:02.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:02 smithi189 bash[23614]: cluster 2024-03-07T03:01:01.229866+0000 mon.a (mon.0) 388 : cluster 0 osdmap e21: 3 total, 3 up, 3 in 2024-03-07T03:01:02.779 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:01:03.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:03 smithi145 bash[30906]: cluster 2024-03-07T03:01:01.274228+0000 mgr.y (mgr.14152) 168 : cluster 0 pgmap v138: 1 pgs: 1 creating+activating; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:03.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:03 smithi145 bash[30906]: cluster 2024-03-07T03:01:02.245583+0000 mon.a (mon.0) 389 : cluster 0 osdmap e22: 3 total, 3 up, 3 in 2024-03-07T03:01:03.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:03 smithi145 bash[30906]: audit 2024-03-07T03:01:03.106871+0000 mon.a (mon.0) 390 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:03.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:03 smithi145 bash[21339]: cluster 2024-03-07T03:01:01.274228+0000 mgr.y (mgr.14152) 168 : cluster 0 pgmap v138: 1 pgs: 1 creating+activating; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:03.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:03 smithi145 bash[21339]: cluster 2024-03-07T03:01:02.245583+0000 mon.a (mon.0) 389 : cluster 0 osdmap e22: 3 total, 3 up, 3 in 2024-03-07T03:01:03.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:03 smithi145 bash[21339]: audit 2024-03-07T03:01:03.106871+0000 mon.a (mon.0) 390 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:03.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:03 smithi189 bash[23614]: cluster 2024-03-07T03:01:01.274228+0000 mgr.y (mgr.14152) 168 : cluster 0 pgmap v138: 1 pgs: 1 creating+activating; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:03.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:03 smithi189 bash[23614]: cluster 2024-03-07T03:01:02.245583+0000 mon.a (mon.0) 389 : cluster 0 osdmap e22: 3 total, 3 up, 3 in 2024-03-07T03:01:03.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:03 smithi189 bash[23614]: audit 2024-03-07T03:01:03.106871+0000 mon.a (mon.0) 390 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:05.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:05 smithi145 bash[30906]: cluster 2024-03-07T03:01:03.274661+0000 mgr.y (mgr.14152) 169 : cluster 0 pgmap v140: 1 pgs: 1 creating+activating; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:05.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:05 smithi145 bash[30906]: cluster 2024-03-07T03:01:04.252378+0000 mon.a (mon.0) 391 : cluster 0 mgrmap e15: y(active, since 4m), standbys: x 2024-03-07T03:01:05.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:05 smithi145 bash[30906]: audit 2024-03-07T03:01:04.906831+0000 mon.a (mon.0) 392 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:01:05.820 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:05 smithi145 bash[21339]: cluster 2024-03-07T03:01:03.274661+0000 mgr.y (mgr.14152) 169 : cluster 0 pgmap v140: 1 pgs: 1 creating+activating; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:05.820 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:05 smithi145 bash[21339]: cluster 2024-03-07T03:01:04.252378+0000 mon.a (mon.0) 391 : cluster 0 mgrmap e15: y(active, since 4m), standbys: x 2024-03-07T03:01:05.820 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:05 smithi145 bash[21339]: audit 2024-03-07T03:01:04.906831+0000 mon.a (mon.0) 392 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:01:06.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:05 smithi189 bash[23614]: cluster 2024-03-07T03:01:03.274661+0000 mgr.y (mgr.14152) 169 : cluster 0 pgmap v140: 1 pgs: 1 creating+activating; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:06.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:05 smithi189 bash[23614]: cluster 2024-03-07T03:01:04.252378+0000 mon.a (mon.0) 391 : cluster 0 mgrmap e15: y(active, since 4m), standbys: x 2024-03-07T03:01:06.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:05 smithi189 bash[23614]: audit 2024-03-07T03:01:04.906831+0000 mon.a (mon.0) 392 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:01:07.683 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-07T03:01:07.683 INFO:teuthology.orchestra.run.smithi145.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-07T03:01:07.683 INFO:teuthology.orchestra.run.smithi145.stderr: stderr: 10+0 records in 2024-03-07T03:01:07.683 INFO:teuthology.orchestra.run.smithi145.stderr:10+0 records out 2024-03-07T03:01:07.683 INFO:teuthology.orchestra.run.smithi145.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0152396 s, 688 MB/s 2024-03-07T03:01:07.683 INFO:teuthology.orchestra.run.smithi145.stderr:--> Zapping successful for: 2024-03-07T03:01:07.903 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: cluster 2024-03-07T03:01:05.275158+0000 mgr.y (mgr.14152) 170 : cluster 0 pgmap v141: 1 pgs: 1 creating+activating; 0 B data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:07.903 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: audit 2024-03-07T03:01:05.761669+0000 mon.a (mon.0) 393 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:01:07.903 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: audit 2024-03-07T03:01:05.762678+0000 mon.a (mon.0) 394 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:01:07.904 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: audit 2024-03-07T03:01:05.762830+0000 mon.a (mon.0) 395 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:01:07.904 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: audit 2024-03-07T03:01:05.763022+0000 mon.a (mon.0) 396 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:07.904 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: audit 2024-03-07T03:01:05.775289+0000 mon.a (mon.0) 397 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:01:07.904 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: audit 2024-03-07T03:01:05.775473+0000 mon.a (mon.0) 398 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:01:07.904 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: audit 2024-03-07T03:01:05.775639+0000 mon.a (mon.0) 399 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:07.904 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: audit 2024-03-07T03:01:05.776529+0000 mon.b (mon.1) 8 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:01:07.904 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: audit 2024-03-07T03:01:06.682515+0000 mon.b (mon.1) 9 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:01:07.904 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: audit 2024-03-07T03:01:06.730846+0000 mon.a (mon.0) 400 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:07.904 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: audit 2024-03-07T03:01:06.733535+0000 mon.a (mon.0) 401 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:01:07.904 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: audit 2024-03-07T03:01:06.733734+0000 mon.a (mon.0) 402 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:01:07.904 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[21339]: audit 2024-03-07T03:01:06.733971+0000 mon.a (mon.0) 403 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:07.904 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: cluster 2024-03-07T03:01:05.275158+0000 mgr.y (mgr.14152) 170 : cluster 0 pgmap v141: 1 pgs: 1 creating+activating; 0 B data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:07.904 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: audit 2024-03-07T03:01:05.761669+0000 mon.a (mon.0) 393 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:01:07.905 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: audit 2024-03-07T03:01:05.762678+0000 mon.a (mon.0) 394 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:01:07.905 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: audit 2024-03-07T03:01:05.762830+0000 mon.a (mon.0) 395 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:01:07.905 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: audit 2024-03-07T03:01:05.763022+0000 mon.a (mon.0) 396 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:07.905 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: audit 2024-03-07T03:01:05.775289+0000 mon.a (mon.0) 397 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:01:07.905 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: audit 2024-03-07T03:01:05.775473+0000 mon.a (mon.0) 398 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:01:07.905 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: audit 2024-03-07T03:01:05.775639+0000 mon.a (mon.0) 399 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:07.905 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: audit 2024-03-07T03:01:05.776529+0000 mon.b (mon.1) 8 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:01:07.905 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: audit 2024-03-07T03:01:06.682515+0000 mon.b (mon.1) 9 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:01:07.905 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: audit 2024-03-07T03:01:06.730846+0000 mon.a (mon.0) 400 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:07.905 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: audit 2024-03-07T03:01:06.733535+0000 mon.a (mon.0) 401 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:01:07.905 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: audit 2024-03-07T03:01:06.733734+0000 mon.a (mon.0) 402 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:01:07.905 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:07 smithi145 bash[30906]: audit 2024-03-07T03:01:06.733971+0000 mon.a (mon.0) 403 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:08.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: cluster 2024-03-07T03:01:05.275158+0000 mgr.y (mgr.14152) 170 : cluster 0 pgmap v141: 1 pgs: 1 creating+activating; 0 B data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:08.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: audit 2024-03-07T03:01:05.761669+0000 mon.a (mon.0) 393 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:01:08.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: audit 2024-03-07T03:01:05.762678+0000 mon.a (mon.0) 394 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:01:08.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: audit 2024-03-07T03:01:05.762830+0000 mon.a (mon.0) 395 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:01:08.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: audit 2024-03-07T03:01:05.763022+0000 mon.a (mon.0) 396 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:08.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: audit 2024-03-07T03:01:05.775289+0000 mon.a (mon.0) 397 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:01:08.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: audit 2024-03-07T03:01:05.775473+0000 mon.a (mon.0) 398 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:01:08.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: audit 2024-03-07T03:01:05.775639+0000 mon.a (mon.0) 399 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:08.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: audit 2024-03-07T03:01:05.776529+0000 mon.b (mon.1) 8 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:01:08.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: audit 2024-03-07T03:01:06.682515+0000 mon.b (mon.1) 9 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:01:08.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: audit 2024-03-07T03:01:06.730846+0000 mon.a (mon.0) 400 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:08.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: audit 2024-03-07T03:01:06.733535+0000 mon.a (mon.0) 401 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:01:08.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: audit 2024-03-07T03:01:06.733734+0000 mon.a (mon.0) 402 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:01:08.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:07 smithi189 bash[23614]: audit 2024-03-07T03:01:06.733971+0000 mon.a (mon.0) 403 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:08.932 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:08 smithi145 bash[30906]: audit 2024-03-07T03:01:06.733290+0000 mon.c (mon.2) 5 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:01:08.932 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:08 smithi145 bash[30906]: cluster 2024-03-07T03:01:07.275539+0000 mgr.y (mgr.14152) 171 : cluster 0 pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:08.932 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:08 smithi145 bash[30906]: audit 2024-03-07T03:01:07.607580+0000 mon.c (mon.2) 6 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:01:08.932 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:08 smithi145 bash[21339]: audit 2024-03-07T03:01:06.733290+0000 mon.c (mon.2) 5 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:01:08.932 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:08 smithi145 bash[21339]: cluster 2024-03-07T03:01:07.275539+0000 mgr.y (mgr.14152) 171 : cluster 0 pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:08.932 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:08 smithi145 bash[21339]: audit 2024-03-07T03:01:07.607580+0000 mon.c (mon.2) 6 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:01:08.933 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi145:vg_nvme/lv_1 2024-03-07T03:01:09.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:08 smithi189 bash[23614]: audit 2024-03-07T03:01:06.733290+0000 mon.c (mon.2) 5 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:01:09.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:08 smithi189 bash[23614]: cluster 2024-03-07T03:01:07.275539+0000 mgr.y (mgr.14152) 171 : cluster 0 pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:09.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:08 smithi189 bash[23614]: audit 2024-03-07T03:01:07.607580+0000 mon.c (mon.2) 6 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:01:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:10 smithi189 bash[23614]: cephadm 2024-03-07T03:01:09.076905+0000 mgr.y (mgr.14152) 172 : cephadm 1 Detected new or changed devices on smithi145 2024-03-07T03:01:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:10 smithi189 bash[23614]: audit 2024-03-07T03:01:09.084294+0000 mon.a (mon.0) 404 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:10 smithi189 bash[23614]: audit 2024-03-07T03:01:09.090812+0000 mon.a (mon.0) 405 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:10 smithi189 bash[23614]: audit 2024-03-07T03:01:09.091899+0000 mon.a (mon.0) 406 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:01:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:10 smithi189 bash[23614]: audit 2024-03-07T03:01:09.092563+0000 mon.a (mon.0) 407 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:01:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:10 smithi189 bash[23614]: audit 2024-03-07T03:01:09.093167+0000 mon.a (mon.0) 408 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:01:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:10 smithi189 bash[23614]: cephadm 2024-03-07T03:01:09.093626+0000 mgr.y (mgr.14152) 173 : cephadm 1 Adjusting osd_memory_target on smithi145 to 5413M 2024-03-07T03:01:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:10 smithi189 bash[23614]: audit 2024-03-07T03:01:09.099717+0000 mon.a (mon.0) 409 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:10 smithi189 bash[23614]: audit 2024-03-07T03:01:09.100763+0000 mon.a (mon.0) 410 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:10 smithi189 bash[23614]: audit 2024-03-07T03:01:09.101463+0000 mon.a (mon.0) 411 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:10 smithi189 bash[23614]: audit 2024-03-07T03:01:09.108487+0000 mon.a (mon.0) 412 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:10.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[21339]: cephadm 2024-03-07T03:01:09.076905+0000 mgr.y (mgr.14152) 172 : cephadm 1 Detected new or changed devices on smithi145 2024-03-07T03:01:10.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[21339]: audit 2024-03-07T03:01:09.084294+0000 mon.a (mon.0) 404 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:10.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[21339]: audit 2024-03-07T03:01:09.090812+0000 mon.a (mon.0) 405 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:10.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[21339]: audit 2024-03-07T03:01:09.091899+0000 mon.a (mon.0) 406 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:01:10.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[21339]: audit 2024-03-07T03:01:09.092563+0000 mon.a (mon.0) 407 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:01:10.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[21339]: audit 2024-03-07T03:01:09.093167+0000 mon.a (mon.0) 408 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:01:10.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[21339]: cephadm 2024-03-07T03:01:09.093626+0000 mgr.y (mgr.14152) 173 : cephadm 1 Adjusting osd_memory_target on smithi145 to 5413M 2024-03-07T03:01:10.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[21339]: audit 2024-03-07T03:01:09.099717+0000 mon.a (mon.0) 409 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:10.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[21339]: audit 2024-03-07T03:01:09.100763+0000 mon.a (mon.0) 410 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:10.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[21339]: audit 2024-03-07T03:01:09.101463+0000 mon.a (mon.0) 411 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:10.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[21339]: audit 2024-03-07T03:01:09.108487+0000 mon.a (mon.0) 412 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:10.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[30906]: cephadm 2024-03-07T03:01:09.076905+0000 mgr.y (mgr.14152) 172 : cephadm 1 Detected new or changed devices on smithi145 2024-03-07T03:01:10.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[30906]: audit 2024-03-07T03:01:09.084294+0000 mon.a (mon.0) 404 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:10.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[30906]: audit 2024-03-07T03:01:09.090812+0000 mon.a (mon.0) 405 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:10.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[30906]: audit 2024-03-07T03:01:09.091899+0000 mon.a (mon.0) 406 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:01:10.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[30906]: audit 2024-03-07T03:01:09.092563+0000 mon.a (mon.0) 407 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:01:10.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[30906]: audit 2024-03-07T03:01:09.093167+0000 mon.a (mon.0) 408 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:01:10.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[30906]: cephadm 2024-03-07T03:01:09.093626+0000 mgr.y (mgr.14152) 173 : cephadm 1 Adjusting osd_memory_target on smithi145 to 5413M 2024-03-07T03:01:10.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[30906]: audit 2024-03-07T03:01:09.099717+0000 mon.a (mon.0) 409 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:10.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[30906]: audit 2024-03-07T03:01:09.100763+0000 mon.a (mon.0) 410 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:10.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[30906]: audit 2024-03-07T03:01:09.101463+0000 mon.a (mon.0) 411 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:10.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:10 smithi145 bash[30906]: audit 2024-03-07T03:01:09.108487+0000 mon.a (mon.0) 412 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:11.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:11 smithi145 bash[21339]: cluster 2024-03-07T03:01:09.276031+0000 mgr.y (mgr.14152) 174 : cluster 0 pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:11.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:11 smithi145 bash[30906]: cluster 2024-03-07T03:01:09.276031+0000 mgr.y (mgr.14152) 174 : cluster 0 pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:11.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:11 smithi189 bash[23614]: cluster 2024-03-07T03:01:09.276031+0000 mgr.y (mgr.14152) 174 : cluster 0 pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:13.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:13 smithi145 bash[21339]: cluster 2024-03-07T03:01:11.276591+0000 mgr.y (mgr.14152) 175 : cluster 0 pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-03-07T03:01:13.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:13 smithi145 bash[30906]: cluster 2024-03-07T03:01:11.276591+0000 mgr.y (mgr.14152) 175 : cluster 0 pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-03-07T03:01:13.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:13 smithi189 bash[23614]: cluster 2024-03-07T03:01:11.276591+0000 mgr.y (mgr.14152) 175 : cluster 0 pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-03-07T03:01:13.643 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:01:15.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:15 smithi145 bash[21339]: cluster 2024-03-07T03:01:13.277162+0000 mgr.y (mgr.14152) 176 : cluster 0 pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-07T03:01:15.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:15 smithi145 bash[30906]: cluster 2024-03-07T03:01:13.277162+0000 mgr.y (mgr.14152) 176 : cluster 0 pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-07T03:01:15.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:15 smithi189 bash[23614]: cluster 2024-03-07T03:01:13.277162+0000 mgr.y (mgr.14152) 176 : cluster 0 pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-07T03:01:17.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:17 smithi145 bash[21339]: cluster 2024-03-07T03:01:15.277695+0000 mgr.y (mgr.14152) 177 : cluster 0 pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-07T03:01:17.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:17 smithi145 bash[21339]: audit 2024-03-07T03:01:16.770495+0000 mon.a (mon.0) 413 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:01:17.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:17 smithi145 bash[21339]: audit 2024-03-07T03:01:16.776136+0000 mon.a (mon.0) 414 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:01:17.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:17 smithi145 bash[21339]: audit 2024-03-07T03:01:16.777454+0000 mon.a (mon.0) 415 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:17.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:17 smithi145 bash[30906]: cluster 2024-03-07T03:01:15.277695+0000 mgr.y (mgr.14152) 177 : cluster 0 pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-07T03:01:17.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:17 smithi145 bash[30906]: audit 2024-03-07T03:01:16.770495+0000 mon.a (mon.0) 413 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:01:17.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:17 smithi145 bash[30906]: audit 2024-03-07T03:01:16.776136+0000 mon.a (mon.0) 414 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:01:17.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:17 smithi145 bash[30906]: audit 2024-03-07T03:01:16.777454+0000 mon.a (mon.0) 415 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:17.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:17 smithi189 bash[23614]: cluster 2024-03-07T03:01:15.277695+0000 mgr.y (mgr.14152) 177 : cluster 0 pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-07T03:01:17.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:17 smithi189 bash[23614]: audit 2024-03-07T03:01:16.770495+0000 mon.a (mon.0) 413 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:01:17.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:17 smithi189 bash[23614]: audit 2024-03-07T03:01:16.776136+0000 mon.a (mon.0) 414 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:01:17.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:17 smithi189 bash[23614]: audit 2024-03-07T03:01:16.777454+0000 mon.a (mon.0) 415 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:18.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:18 smithi145 bash[21339]: audit 2024-03-07T03:01:16.763825+0000 mgr.y (mgr.14152) 178 : audit 0 from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:01:18.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:18 smithi145 bash[30906]: audit 2024-03-07T03:01:16.763825+0000 mgr.y (mgr.14152) 178 : audit 0 from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:01:18.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:18 smithi189 bash[23614]: audit 2024-03-07T03:01:16.763825+0000 mgr.y (mgr.14152) 178 : audit 0 from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi145:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:01:19.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:19 smithi145 bash[21339]: cluster 2024-03-07T03:01:17.278294+0000 mgr.y (mgr.14152) 179 : cluster 0 pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-07T03:01:19.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:19 smithi145 bash[30906]: cluster 2024-03-07T03:01:17.278294+0000 mgr.y (mgr.14152) 179 : cluster 0 pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-07T03:01:19.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:19 smithi189 bash[23614]: cluster 2024-03-07T03:01:17.278294+0000 mgr.y (mgr.14152) 179 : cluster 0 pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-07T03:01:21.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:21 smithi145 bash[21339]: cluster 2024-03-07T03:01:19.278803+0000 mgr.y (mgr.14152) 180 : cluster 0 pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:01:21.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:21 smithi145 bash[30906]: cluster 2024-03-07T03:01:19.278803+0000 mgr.y (mgr.14152) 180 : cluster 0 pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:01:21.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:21 smithi189 bash[23614]: cluster 2024-03-07T03:01:19.278803+0000 mgr.y (mgr.14152) 180 : cluster 0 pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:01:23.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:23 smithi145 bash[21339]: cluster 2024-03-07T03:01:21.279317+0000 mgr.y (mgr.14152) 181 : cluster 0 pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:01:23.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:23 smithi145 bash[30906]: cluster 2024-03-07T03:01:21.279317+0000 mgr.y (mgr.14152) 181 : cluster 0 pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:01:23.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:23 smithi189 bash[23614]: cluster 2024-03-07T03:01:21.279317+0000 mgr.y (mgr.14152) 181 : cluster 0 pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:01:25.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:25 smithi145 bash[30906]: cluster 2024-03-07T03:01:23.279849+0000 mgr.y (mgr.14152) 182 : cluster 0 pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:25.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:25 smithi145 bash[21339]: cluster 2024-03-07T03:01:23.279849+0000 mgr.y (mgr.14152) 182 : cluster 0 pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:25.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:25 smithi189 bash[23614]: cluster 2024-03-07T03:01:23.279849+0000 mgr.y (mgr.14152) 182 : cluster 0 pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:26.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:26 smithi145 bash[21339]: audit 2024-03-07T03:01:25.320951+0000 mon.c (mon.2) 7 : audit 1 from='client.? 172.21.15.145:0/2971094985' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f466751-f5a7-4d18-87ab-96251b77b663"}]: dispatch 2024-03-07T03:01:26.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:26 smithi145 bash[21339]: audit 2024-03-07T03:01:25.321466+0000 mon.a (mon.0) 416 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f466751-f5a7-4d18-87ab-96251b77b663"}]: dispatch 2024-03-07T03:01:26.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:26 smithi145 bash[21339]: audit 2024-03-07T03:01:25.327673+0000 mon.a (mon.0) 417 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f466751-f5a7-4d18-87ab-96251b77b663"}]': finished 2024-03-07T03:01:26.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:26 smithi145 bash[21339]: cluster 2024-03-07T03:01:25.327714+0000 mon.a (mon.0) 418 : cluster 0 osdmap e23: 4 total, 3 up, 4 in 2024-03-07T03:01:26.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:26 smithi145 bash[21339]: audit 2024-03-07T03:01:25.327813+0000 mon.a (mon.0) 419 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:26.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:26 smithi145 bash[21339]: audit 2024-03-07T03:01:26.045966+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.145:0/1977781020' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:01:26.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:26 smithi145 bash[30906]: audit 2024-03-07T03:01:25.320951+0000 mon.c (mon.2) 7 : audit 1 from='client.? 172.21.15.145:0/2971094985' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f466751-f5a7-4d18-87ab-96251b77b663"}]: dispatch 2024-03-07T03:01:26.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:26 smithi145 bash[30906]: audit 2024-03-07T03:01:25.321466+0000 mon.a (mon.0) 416 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f466751-f5a7-4d18-87ab-96251b77b663"}]: dispatch 2024-03-07T03:01:26.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:26 smithi145 bash[30906]: audit 2024-03-07T03:01:25.327673+0000 mon.a (mon.0) 417 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f466751-f5a7-4d18-87ab-96251b77b663"}]': finished 2024-03-07T03:01:26.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:26 smithi145 bash[30906]: cluster 2024-03-07T03:01:25.327714+0000 mon.a (mon.0) 418 : cluster 0 osdmap e23: 4 total, 3 up, 4 in 2024-03-07T03:01:26.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:26 smithi145 bash[30906]: audit 2024-03-07T03:01:25.327813+0000 mon.a (mon.0) 419 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:26.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:26 smithi145 bash[30906]: audit 2024-03-07T03:01:26.045966+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.145:0/1977781020' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:01:26.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:26 smithi189 bash[23614]: audit 2024-03-07T03:01:25.320951+0000 mon.c (mon.2) 7 : audit 1 from='client.? 172.21.15.145:0/2971094985' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f466751-f5a7-4d18-87ab-96251b77b663"}]: dispatch 2024-03-07T03:01:26.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:26 smithi189 bash[23614]: audit 2024-03-07T03:01:25.321466+0000 mon.a (mon.0) 416 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5f466751-f5a7-4d18-87ab-96251b77b663"}]: dispatch 2024-03-07T03:01:26.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:26 smithi189 bash[23614]: audit 2024-03-07T03:01:25.327673+0000 mon.a (mon.0) 417 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5f466751-f5a7-4d18-87ab-96251b77b663"}]': finished 2024-03-07T03:01:26.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:26 smithi189 bash[23614]: cluster 2024-03-07T03:01:25.327714+0000 mon.a (mon.0) 418 : cluster 0 osdmap e23: 4 total, 3 up, 4 in 2024-03-07T03:01:26.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:26 smithi189 bash[23614]: audit 2024-03-07T03:01:25.327813+0000 mon.a (mon.0) 419 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:26.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:26 smithi189 bash[23614]: audit 2024-03-07T03:01:26.045966+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.145:0/1977781020' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:01:27.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:27 smithi145 bash[21339]: cluster 2024-03-07T03:01:25.280333+0000 mgr.y (mgr.14152) 183 : cluster 0 pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:27.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:27 smithi145 bash[30906]: cluster 2024-03-07T03:01:25.280333+0000 mgr.y (mgr.14152) 183 : cluster 0 pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:27.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:27 smithi189 bash[23614]: cluster 2024-03-07T03:01:25.280333+0000 mgr.y (mgr.14152) 183 : cluster 0 pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:29.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:29 smithi145 bash[21339]: cluster 2024-03-07T03:01:27.280898+0000 mgr.y (mgr.14152) 184 : cluster 0 pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:29.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:29 smithi145 bash[30906]: cluster 2024-03-07T03:01:27.280898+0000 mgr.y (mgr.14152) 184 : cluster 0 pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:29.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:29 smithi189 bash[23614]: cluster 2024-03-07T03:01:27.280898+0000 mgr.y (mgr.14152) 184 : cluster 0 pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:30.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:30 smithi145 bash[21339]: cluster 2024-03-07T03:01:29.281440+0000 mgr.y (mgr.14152) 185 : cluster 0 pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:30.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:30 smithi145 bash[30906]: cluster 2024-03-07T03:01:29.281440+0000 mgr.y (mgr.14152) 185 : cluster 0 pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:30.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:30 smithi189 bash[23614]: cluster 2024-03-07T03:01:29.281440+0000 mgr.y (mgr.14152) 185 : cluster 0 pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:32.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:32 smithi145 bash[21339]: cluster 2024-03-07T03:01:31.281928+0000 mgr.y (mgr.14152) 186 : cluster 0 pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:32.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:32 smithi145 bash[30906]: cluster 2024-03-07T03:01:31.281928+0000 mgr.y (mgr.14152) 186 : cluster 0 pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:32 smithi189 bash[23614]: cluster 2024-03-07T03:01:31.281928+0000 mgr.y (mgr.14152) 186 : cluster 0 pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:35.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:35 smithi145 bash[21339]: cluster 2024-03-07T03:01:33.282447+0000 mgr.y (mgr.14152) 187 : cluster 0 pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:35.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:35 smithi145 bash[30906]: cluster 2024-03-07T03:01:33.282447+0000 mgr.y (mgr.14152) 187 : cluster 0 pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:35.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:35 smithi189 bash[23614]: cluster 2024-03-07T03:01:33.282447+0000 mgr.y (mgr.14152) 187 : cluster 0 pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:37.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:37 smithi145 bash[21339]: cluster 2024-03-07T03:01:35.282871+0000 mgr.y (mgr.14152) 188 : cluster 0 pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:37.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:37 smithi145 bash[30906]: cluster 2024-03-07T03:01:35.282871+0000 mgr.y (mgr.14152) 188 : cluster 0 pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:37.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:37 smithi189 bash[23614]: cluster 2024-03-07T03:01:35.282871+0000 mgr.y (mgr.14152) 188 : cluster 0 pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:38.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:38 smithi145 bash[21339]: audit 2024-03-07T03:01:37.952950+0000 mon.a (mon.0) 420 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:01:38.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:38 smithi145 bash[21339]: audit 2024-03-07T03:01:37.954395+0000 mon.a (mon.0) 421 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:38.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:38 smithi145 bash[30906]: audit 2024-03-07T03:01:37.952950+0000 mon.a (mon.0) 420 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:01:38.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:38 smithi145 bash[30906]: audit 2024-03-07T03:01:37.954395+0000 mon.a (mon.0) 421 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:38.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:38 smithi189 bash[23614]: audit 2024-03-07T03:01:37.952950+0000 mon.a (mon.0) 420 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:01:38.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:38 smithi189 bash[23614]: audit 2024-03-07T03:01:37.954395+0000 mon.a (mon.0) 421 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:39.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:39 smithi145 bash[21339]: cluster 2024-03-07T03:01:37.283182+0000 mgr.y (mgr.14152) 189 : cluster 0 pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:39.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:39 smithi145 bash[21339]: cephadm 2024-03-07T03:01:37.955513+0000 mgr.y (mgr.14152) 190 : cephadm 1 Deploying daemon osd.3 on smithi145 2024-03-07T03:01:39.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:39 smithi145 bash[30906]: cluster 2024-03-07T03:01:37.283182+0000 mgr.y (mgr.14152) 189 : cluster 0 pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:39.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:39 smithi145 bash[30906]: cephadm 2024-03-07T03:01:37.955513+0000 mgr.y (mgr.14152) 190 : cephadm 1 Deploying daemon osd.3 on smithi145 2024-03-07T03:01:39.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:39 smithi189 bash[23614]: cluster 2024-03-07T03:01:37.283182+0000 mgr.y (mgr.14152) 189 : cluster 0 pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:39.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:39 smithi189 bash[23614]: cephadm 2024-03-07T03:01:37.955513+0000 mgr.y (mgr.14152) 190 : cephadm 1 Deploying daemon osd.3 on smithi145 2024-03-07T03:01:41.430 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:41 smithi145 bash[21339]: cluster 2024-03-07T03:01:39.283645+0000 mgr.y (mgr.14152) 191 : cluster 0 pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:41.431 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:41 smithi145 bash[30906]: cluster 2024-03-07T03:01:39.283645+0000 mgr.y (mgr.14152) 191 : cluster 0 pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:41.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:41 smithi189 bash[23614]: cluster 2024-03-07T03:01:39.283645+0000 mgr.y (mgr.14152) 191 : cluster 0 pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:42.383 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:42 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:42.384 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:01:42 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:42.384 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:42 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:42.384 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:01:42 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:42.385 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:01:42 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:42.385 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:01:42 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:42.685 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:42 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:42.685 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:42 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:42.686 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:01:42 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:42.686 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:01:42 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:42.686 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:01:42 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:42.686 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:01:42 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:43.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:43 smithi145 bash[21339]: cluster 2024-03-07T03:01:41.284031+0000 mgr.y (mgr.14152) 192 : cluster 0 pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:43.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:43 smithi145 bash[21339]: audit 2024-03-07T03:01:42.672876+0000 mon.a (mon.0) 422 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:43.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:43 smithi145 bash[21339]: audit 2024-03-07T03:01:42.678173+0000 mon.a (mon.0) 423 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:43.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:43 smithi145 bash[21339]: audit 2024-03-07T03:01:42.703770+0000 mon.a (mon.0) 424 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:43.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:43 smithi145 bash[30906]: cluster 2024-03-07T03:01:41.284031+0000 mgr.y (mgr.14152) 192 : cluster 0 pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:43.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:43 smithi145 bash[30906]: audit 2024-03-07T03:01:42.672876+0000 mon.a (mon.0) 422 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:43.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:43 smithi145 bash[30906]: audit 2024-03-07T03:01:42.678173+0000 mon.a (mon.0) 423 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:43.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:43 smithi145 bash[30906]: audit 2024-03-07T03:01:42.703770+0000 mon.a (mon.0) 424 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:43.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:43 smithi189 bash[23614]: cluster 2024-03-07T03:01:41.284031+0000 mgr.y (mgr.14152) 192 : cluster 0 pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:43.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:43 smithi189 bash[23614]: audit 2024-03-07T03:01:42.672876+0000 mon.a (mon.0) 422 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:43.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:43 smithi189 bash[23614]: audit 2024-03-07T03:01:42.678173+0000 mon.a (mon.0) 423 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:43.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:43 smithi189 bash[23614]: audit 2024-03-07T03:01:42.703770+0000 mon.a (mon.0) 424 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:45.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:45 smithi145 bash[21339]: cluster 2024-03-07T03:01:43.284518+0000 mgr.y (mgr.14152) 193 : cluster 0 pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:45.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:45 smithi145 bash[30906]: cluster 2024-03-07T03:01:43.284518+0000 mgr.y (mgr.14152) 193 : cluster 0 pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:45.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:45 smithi189 bash[23614]: cluster 2024-03-07T03:01:43.284518+0000 mgr.y (mgr.14152) 193 : cluster 0 pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:47.517 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:47 smithi145 bash[21339]: cluster 2024-03-07T03:01:45.285057+0000 mgr.y (mgr.14152) 194 : cluster 0 pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:47.517 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:47 smithi145 bash[30906]: cluster 2024-03-07T03:01:45.285057+0000 mgr.y (mgr.14152) 194 : cluster 0 pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:47.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:47 smithi189 bash[23614]: cluster 2024-03-07T03:01:45.285057+0000 mgr.y (mgr.14152) 194 : cluster 0 pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:49.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:48 smithi145 bash[21339]: cluster 2024-03-07T03:01:47.285444+0000 mgr.y (mgr.14152) 195 : cluster 0 pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:49.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:48 smithi145 bash[21339]: audit 2024-03-07T03:01:47.885866+0000 mon.a (mon.0) 425 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:49.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:48 smithi145 bash[21339]: audit 2024-03-07T03:01:47.893177+0000 mon.a (mon.0) 426 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:49.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:48 smithi145 bash[21339]: audit 2024-03-07T03:01:47.894195+0000 mon.a (mon.0) 427 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:49.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:48 smithi145 bash[21339]: audit 2024-03-07T03:01:47.895055+0000 mon.a (mon.0) 428 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:49.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:48 smithi145 bash[21339]: audit 2024-03-07T03:01:47.900232+0000 mon.a (mon.0) 429 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:49.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:48 smithi145 bash[30906]: cluster 2024-03-07T03:01:47.285444+0000 mgr.y (mgr.14152) 195 : cluster 0 pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:49.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:48 smithi145 bash[30906]: audit 2024-03-07T03:01:47.885866+0000 mon.a (mon.0) 425 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:49.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:48 smithi145 bash[30906]: audit 2024-03-07T03:01:47.893177+0000 mon.a (mon.0) 426 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:49.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:48 smithi145 bash[30906]: audit 2024-03-07T03:01:47.894195+0000 mon.a (mon.0) 427 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:49.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:48 smithi145 bash[30906]: audit 2024-03-07T03:01:47.895055+0000 mon.a (mon.0) 428 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:49.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:48 smithi145 bash[30906]: audit 2024-03-07T03:01:47.900232+0000 mon.a (mon.0) 429 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:49.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:48 smithi189 bash[23614]: cluster 2024-03-07T03:01:47.285444+0000 mgr.y (mgr.14152) 195 : cluster 0 pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:49.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:48 smithi189 bash[23614]: audit 2024-03-07T03:01:47.885866+0000 mon.a (mon.0) 425 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:49.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:48 smithi189 bash[23614]: audit 2024-03-07T03:01:47.893177+0000 mon.a (mon.0) 426 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:49.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:48 smithi189 bash[23614]: audit 2024-03-07T03:01:47.894195+0000 mon.a (mon.0) 427 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:49.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:48 smithi189 bash[23614]: audit 2024-03-07T03:01:47.895055+0000 mon.a (mon.0) 428 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:49.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:48 smithi189 bash[23614]: audit 2024-03-07T03:01:47.900232+0000 mon.a (mon.0) 429 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:50.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:49 smithi145 bash[21339]: audit 2024-03-07T03:01:49.394989+0000 mon.a (mon.0) 430 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:01:50.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:49 smithi145 bash[30906]: audit 2024-03-07T03:01:49.394989+0000 mon.a (mon.0) 430 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:01:50.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:49 smithi189 bash[23614]: audit 2024-03-07T03:01:49.394989+0000 mon.a (mon.0) 430 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:01:51.204 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:50 smithi145 bash[21339]: cluster 2024-03-07T03:01:49.285945+0000 mgr.y (mgr.14152) 196 : cluster 0 pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:51.204 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:50 smithi145 bash[21339]: audit 2024-03-07T03:01:49.901160+0000 mon.a (mon.0) 431 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-07T03:01:51.204 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:50 smithi145 bash[21339]: cluster 2024-03-07T03:01:49.901358+0000 mon.a (mon.0) 432 : cluster 0 osdmap e24: 4 total, 3 up, 4 in 2024-03-07T03:01:51.204 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:50 smithi145 bash[21339]: audit 2024-03-07T03:01:49.901647+0000 mon.a (mon.0) 433 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:51.204 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:50 smithi145 bash[21339]: audit 2024-03-07T03:01:49.902357+0000 mon.a (mon.0) 434 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:01:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:50 smithi145 bash[30906]: cluster 2024-03-07T03:01:49.285945+0000 mgr.y (mgr.14152) 196 : cluster 0 pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:50 smithi145 bash[30906]: audit 2024-03-07T03:01:49.901160+0000 mon.a (mon.0) 431 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-07T03:01:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:50 smithi145 bash[30906]: cluster 2024-03-07T03:01:49.901358+0000 mon.a (mon.0) 432 : cluster 0 osdmap e24: 4 total, 3 up, 4 in 2024-03-07T03:01:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:50 smithi145 bash[30906]: audit 2024-03-07T03:01:49.901647+0000 mon.a (mon.0) 433 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:51.204 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:50 smithi145 bash[30906]: audit 2024-03-07T03:01:49.902357+0000 mon.a (mon.0) 434 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:01:51.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:50 smithi189 bash[23614]: cluster 2024-03-07T03:01:49.285945+0000 mgr.y (mgr.14152) 196 : cluster 0 pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:51.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:50 smithi189 bash[23614]: audit 2024-03-07T03:01:49.901160+0000 mon.a (mon.0) 431 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-07T03:01:51.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:50 smithi189 bash[23614]: cluster 2024-03-07T03:01:49.901358+0000 mon.a (mon.0) 432 : cluster 0 osdmap e24: 4 total, 3 up, 4 in 2024-03-07T03:01:51.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:50 smithi189 bash[23614]: audit 2024-03-07T03:01:49.901647+0000 mon.a (mon.0) 433 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:51.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:50 smithi189 bash[23614]: audit 2024-03-07T03:01:49.902357+0000 mon.a (mon.0) 434 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:01:51.819 INFO:teuthology.orchestra.run.smithi145.stdout:Created osd(s) 3 on host 'smithi145' 2024-03-07T03:01:52.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[21339]: audit 2024-03-07T03:01:50.901156+0000 mon.a (mon.0) 435 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-03-07T03:01:52.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[21339]: cluster 2024-03-07T03:01:50.901221+0000 mon.a (mon.0) 436 : cluster 0 osdmap e25: 4 total, 3 up, 4 in 2024-03-07T03:01:52.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[21339]: audit 2024-03-07T03:01:50.901628+0000 mon.a (mon.0) 437 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:52.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[21339]: audit 2024-03-07T03:01:50.906892+0000 mon.a (mon.0) 438 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:52.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[21339]: audit 2024-03-07T03:01:51.793578+0000 mon.a (mon.0) 439 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:52.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[21339]: audit 2024-03-07T03:01:51.805331+0000 mon.a (mon.0) 440 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:52.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[21339]: audit 2024-03-07T03:01:51.823373+0000 mon.a (mon.0) 441 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:52.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[21339]: audit 2024-03-07T03:01:51.906248+0000 mon.a (mon.0) 442 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:52.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[30906]: audit 2024-03-07T03:01:50.901156+0000 mon.a (mon.0) 435 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-03-07T03:01:52.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[30906]: cluster 2024-03-07T03:01:50.901221+0000 mon.a (mon.0) 436 : cluster 0 osdmap e25: 4 total, 3 up, 4 in 2024-03-07T03:01:52.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[30906]: audit 2024-03-07T03:01:50.901628+0000 mon.a (mon.0) 437 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:52.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[30906]: audit 2024-03-07T03:01:50.906892+0000 mon.a (mon.0) 438 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:52.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[30906]: audit 2024-03-07T03:01:51.793578+0000 mon.a (mon.0) 439 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:52.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[30906]: audit 2024-03-07T03:01:51.805331+0000 mon.a (mon.0) 440 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:52.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[30906]: audit 2024-03-07T03:01:51.823373+0000 mon.a (mon.0) 441 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:52.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:51 smithi145 bash[30906]: audit 2024-03-07T03:01:51.906248+0000 mon.a (mon.0) 442 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:52.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:51 smithi189 bash[23614]: audit 2024-03-07T03:01:50.901156+0000 mon.a (mon.0) 435 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]': finished 2024-03-07T03:01:52.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:51 smithi189 bash[23614]: cluster 2024-03-07T03:01:50.901221+0000 mon.a (mon.0) 436 : cluster 0 osdmap e25: 4 total, 3 up, 4 in 2024-03-07T03:01:52.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:51 smithi189 bash[23614]: audit 2024-03-07T03:01:50.901628+0000 mon.a (mon.0) 437 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:52.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:51 smithi189 bash[23614]: audit 2024-03-07T03:01:50.906892+0000 mon.a (mon.0) 438 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:52.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:51 smithi189 bash[23614]: audit 2024-03-07T03:01:51.793578+0000 mon.a (mon.0) 439 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:52.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:51 smithi189 bash[23614]: audit 2024-03-07T03:01:51.805331+0000 mon.a (mon.0) 440 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:01:52.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:51 smithi189 bash[23614]: audit 2024-03-07T03:01:51.823373+0000 mon.a (mon.0) 441 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:52.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:51 smithi189 bash[23614]: audit 2024-03-07T03:01:51.906248+0000 mon.a (mon.0) 442 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:52.547 DEBUG:teuthology.orchestra.run.smithi145:osd.3> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.3.service 2024-03-07T03:01:52.550 INFO:tasks.cephadm:Deploying osd.4 on smithi189 with /dev/vg_nvme/lv_4... 2024-03-07T03:01:52.550 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-07T03:01:53.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:52 smithi145 bash[21339]: cluster 2024-03-07T03:01:50.372656+0000 osd.3 (osd.3) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:01:53.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:52 smithi145 bash[21339]: cluster 2024-03-07T03:01:50.372732+0000 osd.3 (osd.3) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:01:53.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:52 smithi145 bash[21339]: cluster 2024-03-07T03:01:51.286583+0000 mgr.y (mgr.14152) 197 : cluster 0 pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:53.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:52 smithi145 bash[21339]: cluster 2024-03-07T03:01:51.912222+0000 mon.a (mon.0) 443 : cluster 1 osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084] boot 2024-03-07T03:01:53.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:52 smithi145 bash[21339]: cluster 2024-03-07T03:01:51.912255+0000 mon.a (mon.0) 444 : cluster 0 osdmap e26: 4 total, 4 up, 4 in 2024-03-07T03:01:53.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:52 smithi145 bash[21339]: audit 2024-03-07T03:01:51.912873+0000 mon.a (mon.0) 445 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:53.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:52 smithi145 bash[30906]: cluster 2024-03-07T03:01:50.372656+0000 osd.3 (osd.3) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:01:53.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:52 smithi145 bash[30906]: cluster 2024-03-07T03:01:50.372732+0000 osd.3 (osd.3) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:01:53.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:52 smithi145 bash[30906]: cluster 2024-03-07T03:01:51.286583+0000 mgr.y (mgr.14152) 197 : cluster 0 pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:53.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:52 smithi145 bash[30906]: cluster 2024-03-07T03:01:51.912222+0000 mon.a (mon.0) 443 : cluster 1 osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084] boot 2024-03-07T03:01:53.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:52 smithi145 bash[30906]: cluster 2024-03-07T03:01:51.912255+0000 mon.a (mon.0) 444 : cluster 0 osdmap e26: 4 total, 4 up, 4 in 2024-03-07T03:01:53.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:52 smithi145 bash[30906]: audit 2024-03-07T03:01:51.912873+0000 mon.a (mon.0) 445 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:53.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:52 smithi189 bash[23614]: cluster 2024-03-07T03:01:50.372656+0000 osd.3 (osd.3) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:01:53.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:52 smithi189 bash[23614]: cluster 2024-03-07T03:01:50.372732+0000 osd.3 (osd.3) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:01:53.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:52 smithi189 bash[23614]: cluster 2024-03-07T03:01:51.286583+0000 mgr.y (mgr.14152) 197 : cluster 0 pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:01:53.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:52 smithi189 bash[23614]: cluster 2024-03-07T03:01:51.912222+0000 mon.a (mon.0) 443 : cluster 1 osd.3 [v2:172.21.15.145:6826/2306759084,v1:172.21.15.145:6827/2306759084] boot 2024-03-07T03:01:53.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:52 smithi189 bash[23614]: cluster 2024-03-07T03:01:51.912255+0000 mon.a (mon.0) 444 : cluster 0 osdmap e26: 4 total, 4 up, 4 in 2024-03-07T03:01:53.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:52 smithi189 bash[23614]: audit 2024-03-07T03:01:51.912873+0000 mon.a (mon.0) 445 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:01:54.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:53 smithi145 bash[21339]: cluster 2024-03-07T03:01:52.936573+0000 mon.a (mon.0) 446 : cluster 0 osdmap e27: 4 total, 4 up, 4 in 2024-03-07T03:01:54.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:53 smithi145 bash[30906]: cluster 2024-03-07T03:01:52.936573+0000 mon.a (mon.0) 446 : cluster 0 osdmap e27: 4 total, 4 up, 4 in 2024-03-07T03:01:54.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:53 smithi189 bash[23614]: cluster 2024-03-07T03:01:52.936573+0000 mon.a (mon.0) 446 : cluster 0 osdmap e27: 4 total, 4 up, 4 in 2024-03-07T03:01:55.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:55 smithi145 bash[21339]: cluster 2024-03-07T03:01:53.287106+0000 mgr.y (mgr.14152) 198 : cluster 0 pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:01:55.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:55 smithi145 bash[30906]: cluster 2024-03-07T03:01:53.287106+0000 mgr.y (mgr.14152) 198 : cluster 0 pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:01:55.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:55 smithi189 bash[23614]: cluster 2024-03-07T03:01:53.287106+0000 mgr.y (mgr.14152) 198 : cluster 0 pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:01:57.222 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:01:57.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:57 smithi145 bash[21339]: cluster 2024-03-07T03:01:55.287658+0000 mgr.y (mgr.14152) 199 : cluster 0 pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:01:57.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:57 smithi145 bash[30906]: cluster 2024-03-07T03:01:55.287658+0000 mgr.y (mgr.14152) 199 : cluster 0 pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:01:57.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:57 smithi189 bash[23614]: cluster 2024-03-07T03:01:55.287658+0000 mgr.y (mgr.14152) 199 : cluster 0 pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:01:59.552 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:01:59 smithi145 bash[21339]: cluster 2024-03-07T03:01:57.288234+0000 mgr.y (mgr.14152) 200 : cluster 0 pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:01:59.552 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:01:59 smithi145 bash[30906]: cluster 2024-03-07T03:01:57.288234+0000 mgr.y (mgr.14152) 200 : cluster 0 pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:01:59.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:01:59 smithi189 bash[23614]: cluster 2024-03-07T03:01:57.288234+0000 mgr.y (mgr.14152) 200 : cluster 0 pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:00.050 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-07T03:02:00.050 INFO:teuthology.orchestra.run.smithi189.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-07T03:02:00.050 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10+0 records in 2024-03-07T03:02:00.050 INFO:teuthology.orchestra.run.smithi189.stderr:10+0 records out 2024-03-07T03:02:00.050 INFO:teuthology.orchestra.run.smithi189.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0136516 s, 768 MB/s 2024-03-07T03:02:00.051 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping successful for: 2024-03-07T03:02:00.650 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi189:vg_nvme/lv_4 2024-03-07T03:02:01.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:01 smithi145 bash[21339]: cluster 2024-03-07T03:01:59.288780+0000 mgr.y (mgr.14152) 201 : cluster 0 pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:01.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:01 smithi145 bash[30906]: cluster 2024-03-07T03:01:59.288780+0000 mgr.y (mgr.14152) 201 : cluster 0 pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:01.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:01 smithi189 bash[23614]: cluster 2024-03-07T03:01:59.288780+0000 mgr.y (mgr.14152) 201 : cluster 0 pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:02 smithi189 bash[23614]: cluster 2024-03-07T03:02:01.289352+0000 mgr.y (mgr.14152) 202 : cluster 0 pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:02 smithi189 bash[23614]: cephadm 2024-03-07T03:02:01.564666+0000 mgr.y (mgr.14152) 203 : cephadm 1 Detected new or changed devices on smithi145 2024-03-07T03:02:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:02 smithi189 bash[23614]: audit 2024-03-07T03:02:01.576628+0000 mon.a (mon.0) 447 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:02 smithi189 bash[23614]: audit 2024-03-07T03:02:01.589143+0000 mon.a (mon.0) 448 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:02 smithi189 bash[23614]: audit 2024-03-07T03:02:01.592071+0000 mon.a (mon.0) 449 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:02 smithi189 bash[23614]: audit 2024-03-07T03:02:01.594199+0000 mon.a (mon.0) 450 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:02 smithi189 bash[23614]: audit 2024-03-07T03:02:01.596156+0000 mon.a (mon.0) 451 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:02 smithi189 bash[23614]: audit 2024-03-07T03:02:01.598022+0000 mon.a (mon.0) 452 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:02 smithi189 bash[23614]: cephadm 2024-03-07T03:02:01.599542+0000 mgr.y (mgr.14152) 204 : cephadm 1 Adjusting osd_memory_target on smithi145 to 4060M 2024-03-07T03:02:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:02 smithi189 bash[23614]: audit 2024-03-07T03:02:01.609324+0000 mon.a (mon.0) 453 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:02.840 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:02 smithi189 bash[23614]: audit 2024-03-07T03:02:01.611993+0000 mon.a (mon.0) 454 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:02.840 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:02 smithi189 bash[23614]: audit 2024-03-07T03:02:01.613700+0000 mon.a (mon.0) 455 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:02.840 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:02 smithi189 bash[23614]: audit 2024-03-07T03:02:01.624091+0000 mon.a (mon.0) 456 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:03.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[21339]: cluster 2024-03-07T03:02:01.289352+0000 mgr.y (mgr.14152) 202 : cluster 0 pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[21339]: cephadm 2024-03-07T03:02:01.564666+0000 mgr.y (mgr.14152) 203 : cephadm 1 Detected new or changed devices on smithi145 2024-03-07T03:02:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[21339]: audit 2024-03-07T03:02:01.576628+0000 mon.a (mon.0) 447 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[21339]: audit 2024-03-07T03:02:01.589143+0000 mon.a (mon.0) 448 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[21339]: audit 2024-03-07T03:02:01.592071+0000 mon.a (mon.0) 449 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[21339]: audit 2024-03-07T03:02:01.594199+0000 mon.a (mon.0) 450 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[21339]: audit 2024-03-07T03:02:01.596156+0000 mon.a (mon.0) 451 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[21339]: audit 2024-03-07T03:02:01.598022+0000 mon.a (mon.0) 452 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[21339]: cephadm 2024-03-07T03:02:01.599542+0000 mgr.y (mgr.14152) 204 : cephadm 1 Adjusting osd_memory_target on smithi145 to 4060M 2024-03-07T03:02:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[21339]: audit 2024-03-07T03:02:01.609324+0000 mon.a (mon.0) 453 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[21339]: audit 2024-03-07T03:02:01.611993+0000 mon.a (mon.0) 454 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[21339]: audit 2024-03-07T03:02:01.613700+0000 mon.a (mon.0) 455 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[21339]: audit 2024-03-07T03:02:01.624091+0000 mon.a (mon.0) 456 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[30906]: cluster 2024-03-07T03:02:01.289352+0000 mgr.y (mgr.14152) 202 : cluster 0 pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[30906]: cephadm 2024-03-07T03:02:01.564666+0000 mgr.y (mgr.14152) 203 : cephadm 1 Detected new or changed devices on smithi145 2024-03-07T03:02:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[30906]: audit 2024-03-07T03:02:01.576628+0000 mon.a (mon.0) 447 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[30906]: audit 2024-03-07T03:02:01.589143+0000 mon.a (mon.0) 448 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[30906]: audit 2024-03-07T03:02:01.592071+0000 mon.a (mon.0) 449 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[30906]: audit 2024-03-07T03:02:01.594199+0000 mon.a (mon.0) 450 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[30906]: audit 2024-03-07T03:02:01.596156+0000 mon.a (mon.0) 451 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[30906]: audit 2024-03-07T03:02:01.598022+0000 mon.a (mon.0) 452 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[30906]: cephadm 2024-03-07T03:02:01.599542+0000 mgr.y (mgr.14152) 204 : cephadm 1 Adjusting osd_memory_target on smithi145 to 4060M 2024-03-07T03:02:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[30906]: audit 2024-03-07T03:02:01.609324+0000 mon.a (mon.0) 453 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[30906]: audit 2024-03-07T03:02:01.611993+0000 mon.a (mon.0) 454 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[30906]: audit 2024-03-07T03:02:01.613700+0000 mon.a (mon.0) 455 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:02 smithi145 bash[30906]: audit 2024-03-07T03:02:01.624091+0000 mon.a (mon.0) 456 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:05.320 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:02:05.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:05 smithi145 bash[21339]: cluster 2024-03-07T03:02:03.289822+0000 mgr.y (mgr.14152) 205 : cluster 0 pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:05.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:05 smithi145 bash[30906]: cluster 2024-03-07T03:02:03.289822+0000 mgr.y (mgr.14152) 205 : cluster 0 pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:05.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:05 smithi189 bash[23614]: cluster 2024-03-07T03:02:03.289822+0000 mgr.y (mgr.14152) 205 : cluster 0 pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:07.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:07 smithi145 bash[21339]: cluster 2024-03-07T03:02:05.290321+0000 mgr.y (mgr.14152) 206 : cluster 0 pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:07.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:07 smithi145 bash[30906]: cluster 2024-03-07T03:02:05.290321+0000 mgr.y (mgr.14152) 206 : cluster 0 pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:07.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:07 smithi189 bash[23614]: cluster 2024-03-07T03:02:05.290321+0000 mgr.y (mgr.14152) 206 : cluster 0 pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:08.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:08 smithi145 bash[21339]: audit 2024-03-07T03:02:07.706118+0000 mon.a (mon.0) 457 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:02:08.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:08 smithi145 bash[21339]: audit 2024-03-07T03:02:07.712617+0000 mon.a (mon.0) 458 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:02:08.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:08 smithi145 bash[21339]: audit 2024-03-07T03:02:07.714210+0000 mon.a (mon.0) 459 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:08.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:08 smithi145 bash[30906]: audit 2024-03-07T03:02:07.706118+0000 mon.a (mon.0) 457 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:02:08.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:08 smithi145 bash[30906]: audit 2024-03-07T03:02:07.712617+0000 mon.a (mon.0) 458 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:02:08.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:08 smithi145 bash[30906]: audit 2024-03-07T03:02:07.714210+0000 mon.a (mon.0) 459 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:08.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:08 smithi189 bash[23614]: audit 2024-03-07T03:02:07.706118+0000 mon.a (mon.0) 457 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:02:08.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:08 smithi189 bash[23614]: audit 2024-03-07T03:02:07.712617+0000 mon.a (mon.0) 458 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:02:08.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:08 smithi189 bash[23614]: audit 2024-03-07T03:02:07.714210+0000 mon.a (mon.0) 459 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:09.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:09 smithi145 bash[21339]: cluster 2024-03-07T03:02:07.290826+0000 mgr.y (mgr.14152) 207 : cluster 0 pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:09.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:09 smithi145 bash[21339]: audit 2024-03-07T03:02:07.698761+0000 mgr.y (mgr.14152) 208 : audit 0 from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:02:09.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:09 smithi145 bash[30906]: cluster 2024-03-07T03:02:07.290826+0000 mgr.y (mgr.14152) 207 : cluster 0 pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:09.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:09 smithi145 bash[30906]: audit 2024-03-07T03:02:07.698761+0000 mgr.y (mgr.14152) 208 : audit 0 from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:02:09.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:09 smithi189 bash[23614]: cluster 2024-03-07T03:02:07.290826+0000 mgr.y (mgr.14152) 207 : cluster 0 pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:09.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:09 smithi189 bash[23614]: audit 2024-03-07T03:02:07.698761+0000 mgr.y (mgr.14152) 208 : audit 0 from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:02:11.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:11 smithi145 bash[21339]: cluster 2024-03-07T03:02:09.291352+0000 mgr.y (mgr.14152) 209 : cluster 0 pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:11.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:11 smithi145 bash[30906]: cluster 2024-03-07T03:02:09.291352+0000 mgr.y (mgr.14152) 209 : cluster 0 pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:11.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:11 smithi189 bash[23614]: cluster 2024-03-07T03:02:09.291352+0000 mgr.y (mgr.14152) 209 : cluster 0 pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:13.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:13 smithi189 bash[23614]: cluster 2024-03-07T03:02:11.291878+0000 mgr.y (mgr.14152) 210 : cluster 0 pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:13.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:13 smithi145 bash[21339]: cluster 2024-03-07T03:02:11.291878+0000 mgr.y (mgr.14152) 210 : cluster 0 pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:13.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:13 smithi145 bash[30906]: cluster 2024-03-07T03:02:11.291878+0000 mgr.y (mgr.14152) 210 : cluster 0 pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:15.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:15 smithi145 bash[30906]: cluster 2024-03-07T03:02:13.292451+0000 mgr.y (mgr.14152) 211 : cluster 0 pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:15.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:15 smithi145 bash[21339]: cluster 2024-03-07T03:02:13.292451+0000 mgr.y (mgr.14152) 211 : cluster 0 pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:15.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:15 smithi189 bash[23614]: cluster 2024-03-07T03:02:13.292451+0000 mgr.y (mgr.14152) 211 : cluster 0 pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:16 smithi145 bash[30906]: audit 2024-03-07T03:02:15.608742+0000 mon.a (mon.0) 460 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "633f56af-3ff8-4954-97b4-bc8122bb10ee"}]: dispatch 2024-03-07T03:02:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:16 smithi145 bash[30906]: audit 2024-03-07T03:02:15.608761+0000 mon.b (mon.1) 10 : audit 1 from='client.? 172.21.15.189:0/18943987' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "633f56af-3ff8-4954-97b4-bc8122bb10ee"}]: dispatch 2024-03-07T03:02:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:16 smithi145 bash[30906]: audit 2024-03-07T03:02:15.618078+0000 mon.a (mon.0) 461 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "633f56af-3ff8-4954-97b4-bc8122bb10ee"}]': finished 2024-03-07T03:02:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:16 smithi145 bash[30906]: cluster 2024-03-07T03:02:15.618282+0000 mon.a (mon.0) 462 : cluster 0 osdmap e28: 5 total, 4 up, 5 in 2024-03-07T03:02:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:16 smithi145 bash[30906]: audit 2024-03-07T03:02:15.618555+0000 mon.a (mon.0) 463 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:16.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:16 smithi145 bash[21339]: audit 2024-03-07T03:02:15.608742+0000 mon.a (mon.0) 460 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "633f56af-3ff8-4954-97b4-bc8122bb10ee"}]: dispatch 2024-03-07T03:02:16.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:16 smithi145 bash[21339]: audit 2024-03-07T03:02:15.608761+0000 mon.b (mon.1) 10 : audit 1 from='client.? 172.21.15.189:0/18943987' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "633f56af-3ff8-4954-97b4-bc8122bb10ee"}]: dispatch 2024-03-07T03:02:16.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:16 smithi145 bash[21339]: audit 2024-03-07T03:02:15.618078+0000 mon.a (mon.0) 461 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "633f56af-3ff8-4954-97b4-bc8122bb10ee"}]': finished 2024-03-07T03:02:16.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:16 smithi145 bash[21339]: cluster 2024-03-07T03:02:15.618282+0000 mon.a (mon.0) 462 : cluster 0 osdmap e28: 5 total, 4 up, 5 in 2024-03-07T03:02:16.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:16 smithi145 bash[21339]: audit 2024-03-07T03:02:15.618555+0000 mon.a (mon.0) 463 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:16.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:16 smithi189 bash[23614]: audit 2024-03-07T03:02:15.608742+0000 mon.a (mon.0) 460 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "633f56af-3ff8-4954-97b4-bc8122bb10ee"}]: dispatch 2024-03-07T03:02:16.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:16 smithi189 bash[23614]: audit 2024-03-07T03:02:15.608761+0000 mon.b (mon.1) 10 : audit 1 from='client.? 172.21.15.189:0/18943987' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "633f56af-3ff8-4954-97b4-bc8122bb10ee"}]: dispatch 2024-03-07T03:02:16.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:16 smithi189 bash[23614]: audit 2024-03-07T03:02:15.618078+0000 mon.a (mon.0) 461 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "633f56af-3ff8-4954-97b4-bc8122bb10ee"}]': finished 2024-03-07T03:02:16.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:16 smithi189 bash[23614]: cluster 2024-03-07T03:02:15.618282+0000 mon.a (mon.0) 462 : cluster 0 osdmap e28: 5 total, 4 up, 5 in 2024-03-07T03:02:16.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:16 smithi189 bash[23614]: audit 2024-03-07T03:02:15.618555+0000 mon.a (mon.0) 463 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:17.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:17 smithi145 bash[30906]: cluster 2024-03-07T03:02:15.292971+0000 mgr.y (mgr.14152) 212 : cluster 0 pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:17.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:17 smithi145 bash[30906]: audit 2024-03-07T03:02:16.328239+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.189:0/3953215712' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:02:17.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:17 smithi145 bash[21339]: cluster 2024-03-07T03:02:15.292971+0000 mgr.y (mgr.14152) 212 : cluster 0 pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:17.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:17 smithi145 bash[21339]: audit 2024-03-07T03:02:16.328239+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.189:0/3953215712' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:02:17.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:17 smithi189 bash[23614]: cluster 2024-03-07T03:02:15.292971+0000 mgr.y (mgr.14152) 212 : cluster 0 pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:17.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:17 smithi189 bash[23614]: audit 2024-03-07T03:02:16.328239+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.189:0/3953215712' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:02:19.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:19 smithi145 bash[30906]: cluster 2024-03-07T03:02:17.293534+0000 mgr.y (mgr.14152) 213 : cluster 0 pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:19.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:19 smithi145 bash[21339]: cluster 2024-03-07T03:02:17.293534+0000 mgr.y (mgr.14152) 213 : cluster 0 pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:19.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:19 smithi189 bash[23614]: cluster 2024-03-07T03:02:17.293534+0000 mgr.y (mgr.14152) 213 : cluster 0 pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:21.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:21 smithi145 bash[21339]: cluster 2024-03-07T03:02:19.294015+0000 mgr.y (mgr.14152) 214 : cluster 0 pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:21.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:21 smithi145 bash[30906]: cluster 2024-03-07T03:02:19.294015+0000 mgr.y (mgr.14152) 214 : cluster 0 pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:21.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:21 smithi189 bash[23614]: cluster 2024-03-07T03:02:19.294015+0000 mgr.y (mgr.14152) 214 : cluster 0 pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:23.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:23 smithi145 bash[21339]: cluster 2024-03-07T03:02:21.294532+0000 mgr.y (mgr.14152) 215 : cluster 0 pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:23.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:23 smithi145 bash[30906]: cluster 2024-03-07T03:02:21.294532+0000 mgr.y (mgr.14152) 215 : cluster 0 pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:23.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:23 smithi189 bash[23614]: cluster 2024-03-07T03:02:21.294532+0000 mgr.y (mgr.14152) 215 : cluster 0 pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:25.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:25 smithi145 bash[21339]: cluster 2024-03-07T03:02:23.295057+0000 mgr.y (mgr.14152) 216 : cluster 0 pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:25.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:25 smithi145 bash[30906]: cluster 2024-03-07T03:02:23.295057+0000 mgr.y (mgr.14152) 216 : cluster 0 pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:25.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:25 smithi189 bash[23614]: cluster 2024-03-07T03:02:23.295057+0000 mgr.y (mgr.14152) 216 : cluster 0 pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:27.526 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:27 smithi189 bash[23614]: cluster 2024-03-07T03:02:25.295583+0000 mgr.y (mgr.14152) 217 : cluster 0 pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:27.526 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:27 smithi189 bash[23614]: audit 2024-03-07T03:02:27.210734+0000 mon.a (mon.0) 464 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:02:27.526 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:27 smithi189 bash[23614]: audit 2024-03-07T03:02:27.212823+0000 mon.a (mon.0) 465 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:27.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:27 smithi145 bash[21339]: cluster 2024-03-07T03:02:25.295583+0000 mgr.y (mgr.14152) 217 : cluster 0 pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:27.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:27 smithi145 bash[21339]: audit 2024-03-07T03:02:27.210734+0000 mon.a (mon.0) 464 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:02:27.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:27 smithi145 bash[21339]: audit 2024-03-07T03:02:27.212823+0000 mon.a (mon.0) 465 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:27.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:27 smithi145 bash[30906]: cluster 2024-03-07T03:02:25.295583+0000 mgr.y (mgr.14152) 217 : cluster 0 pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:27.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:27 smithi145 bash[30906]: audit 2024-03-07T03:02:27.210734+0000 mon.a (mon.0) 464 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:02:27.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:27 smithi145 bash[30906]: audit 2024-03-07T03:02:27.212823+0000 mon.a (mon.0) 465 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:28.526 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:28 smithi189 bash[23614]: cephadm 2024-03-07T03:02:27.214532+0000 mgr.y (mgr.14152) 218 : cephadm 1 Deploying daemon osd.4 on smithi189 2024-03-07T03:02:28.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:28 smithi145 bash[21339]: cephadm 2024-03-07T03:02:27.214532+0000 mgr.y (mgr.14152) 218 : cephadm 1 Deploying daemon osd.4 on smithi189 2024-03-07T03:02:28.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:28 smithi145 bash[30906]: cephadm 2024-03-07T03:02:27.214532+0000 mgr.y (mgr.14152) 218 : cephadm 1 Deploying daemon osd.4 on smithi189 2024-03-07T03:02:29.518 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:29 smithi189 bash[23614]: cluster 2024-03-07T03:02:27.296108+0000 mgr.y (mgr.14152) 219 : cluster 0 pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:29.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:29 smithi145 bash[21339]: cluster 2024-03-07T03:02:27.296108+0000 mgr.y (mgr.14152) 219 : cluster 0 pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:29.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:29 smithi145 bash[30906]: cluster 2024-03-07T03:02:27.296108+0000 mgr.y (mgr.14152) 219 : cluster 0 pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:30.759 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:30 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:02:30.759 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:02:30 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:02:31.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:30 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:02:31.088 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:02:30 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:02:31.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:31 smithi145 bash[21339]: cluster 2024-03-07T03:02:29.296659+0000 mgr.y (mgr.14152) 220 : cluster 0 pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:31.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:31 smithi145 bash[21339]: audit 2024-03-07T03:02:31.001185+0000 mon.a (mon.0) 466 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:31.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:31 smithi145 bash[21339]: audit 2024-03-07T03:02:31.011725+0000 mon.a (mon.0) 467 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:31.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:31 smithi145 bash[21339]: audit 2024-03-07T03:02:31.038979+0000 mon.a (mon.0) 468 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:31.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:31 smithi145 bash[30906]: cluster 2024-03-07T03:02:29.296659+0000 mgr.y (mgr.14152) 220 : cluster 0 pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:31.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:31 smithi145 bash[30906]: audit 2024-03-07T03:02:31.001185+0000 mon.a (mon.0) 466 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:31.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:31 smithi145 bash[30906]: audit 2024-03-07T03:02:31.011725+0000 mon.a (mon.0) 467 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:31.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:31 smithi145 bash[30906]: audit 2024-03-07T03:02:31.038979+0000 mon.a (mon.0) 468 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:31.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:31 smithi189 bash[23614]: cluster 2024-03-07T03:02:29.296659+0000 mgr.y (mgr.14152) 220 : cluster 0 pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:31.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:31 smithi189 bash[23614]: audit 2024-03-07T03:02:31.001185+0000 mon.a (mon.0) 466 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:31.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:31 smithi189 bash[23614]: audit 2024-03-07T03:02:31.011725+0000 mon.a (mon.0) 467 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:31.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:31 smithi189 bash[23614]: audit 2024-03-07T03:02:31.038979+0000 mon.a (mon.0) 468 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:33.542 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:33 smithi189 bash[23614]: cluster 2024-03-07T03:02:31.297195+0000 mgr.y (mgr.14152) 221 : cluster 0 pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:33.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:33 smithi145 bash[21339]: cluster 2024-03-07T03:02:31.297195+0000 mgr.y (mgr.14152) 221 : cluster 0 pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:33.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:33 smithi145 bash[30906]: cluster 2024-03-07T03:02:31.297195+0000 mgr.y (mgr.14152) 221 : cluster 0 pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:35.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:35 smithi145 bash[21339]: cluster 2024-03-07T03:02:33.297741+0000 mgr.y (mgr.14152) 222 : cluster 0 pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:35.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:35 smithi145 bash[30906]: cluster 2024-03-07T03:02:33.297741+0000 mgr.y (mgr.14152) 222 : cluster 0 pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:35.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:35 smithi189 bash[23614]: cluster 2024-03-07T03:02:33.297741+0000 mgr.y (mgr.14152) 222 : cluster 0 pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:37.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:37 smithi189 bash[23614]: cluster 2024-03-07T03:02:35.298328+0000 mgr.y (mgr.14152) 223 : cluster 0 pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:37.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:37 smithi189 bash[23614]: audit 2024-03-07T03:02:36.132090+0000 mon.a (mon.0) 469 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:37.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:37 smithi189 bash[23614]: audit 2024-03-07T03:02:36.141113+0000 mon.a (mon.0) 470 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:37.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:37 smithi189 bash[23614]: audit 2024-03-07T03:02:36.693739+0000 mon.a (mon.0) 471 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:37.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:37 smithi189 bash[23614]: audit 2024-03-07T03:02:36.695746+0000 mon.a (mon.0) 472 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:37.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:37 smithi189 bash[23614]: audit 2024-03-07T03:02:36.705756+0000 mon.a (mon.0) 473 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:37.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:37 smithi189 bash[23614]: audit 2024-03-07T03:02:37.119680+0000 mon.a (mon.0) 474 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:02:37.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:37 smithi189 bash[23614]: audit 2024-03-07T03:02:37.119772+0000 mon.b (mon.1) 12 : audit 1 from='osd.4 [v2:172.21.15.189:6800/3152992669,v1:172.21.15.189:6801/3152992669]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:02:37.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[21339]: cluster 2024-03-07T03:02:35.298328+0000 mgr.y (mgr.14152) 223 : cluster 0 pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:37.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[21339]: audit 2024-03-07T03:02:36.132090+0000 mon.a (mon.0) 469 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:37.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[21339]: audit 2024-03-07T03:02:36.141113+0000 mon.a (mon.0) 470 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:37.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[21339]: audit 2024-03-07T03:02:36.693739+0000 mon.a (mon.0) 471 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:37.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[21339]: audit 2024-03-07T03:02:36.695746+0000 mon.a (mon.0) 472 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:37.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[21339]: audit 2024-03-07T03:02:36.705756+0000 mon.a (mon.0) 473 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:37.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[21339]: audit 2024-03-07T03:02:37.119680+0000 mon.a (mon.0) 474 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:02:37.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[21339]: audit 2024-03-07T03:02:37.119772+0000 mon.b (mon.1) 12 : audit 1 from='osd.4 [v2:172.21.15.189:6800/3152992669,v1:172.21.15.189:6801/3152992669]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:02:37.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[30906]: cluster 2024-03-07T03:02:35.298328+0000 mgr.y (mgr.14152) 223 : cluster 0 pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:37.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[30906]: audit 2024-03-07T03:02:36.132090+0000 mon.a (mon.0) 469 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:37.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[30906]: audit 2024-03-07T03:02:36.141113+0000 mon.a (mon.0) 470 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:37.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[30906]: audit 2024-03-07T03:02:36.693739+0000 mon.a (mon.0) 471 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:37.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[30906]: audit 2024-03-07T03:02:36.695746+0000 mon.a (mon.0) 472 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:37.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[30906]: audit 2024-03-07T03:02:36.705756+0000 mon.a (mon.0) 473 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:37.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[30906]: audit 2024-03-07T03:02:37.119680+0000 mon.a (mon.0) 474 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:02:37.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:37 smithi145 bash[30906]: audit 2024-03-07T03:02:37.119772+0000 mon.b (mon.1) 12 : audit 1 from='osd.4 [v2:172.21.15.189:6800/3152992669,v1:172.21.15.189:6801/3152992669]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:02:38.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:38 smithi189 bash[23614]: cluster 2024-03-07T03:02:37.298839+0000 mgr.y (mgr.14152) 224 : cluster 0 pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:38.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:38 smithi189 bash[23614]: audit 2024-03-07T03:02:37.709247+0000 mon.a (mon.0) 475 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-07T03:02:38.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:38 smithi189 bash[23614]: cluster 2024-03-07T03:02:37.709548+0000 mon.a (mon.0) 476 : cluster 0 osdmap e29: 5 total, 4 up, 5 in 2024-03-07T03:02:38.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:38 smithi189 bash[23614]: audit 2024-03-07T03:02:37.709818+0000 mon.a (mon.0) 477 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:38.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:38 smithi189 bash[23614]: audit 2024-03-07T03:02:37.711408+0000 mon.a (mon.0) 478 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:02:38.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:38 smithi189 bash[23614]: audit 2024-03-07T03:02:37.711421+0000 mon.b (mon.1) 13 : audit 1 from='osd.4 [v2:172.21.15.189:6800/3152992669,v1:172.21.15.189:6801/3152992669]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:02:39.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:38 smithi145 bash[21339]: cluster 2024-03-07T03:02:37.298839+0000 mgr.y (mgr.14152) 224 : cluster 0 pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:39.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:38 smithi145 bash[21339]: audit 2024-03-07T03:02:37.709247+0000 mon.a (mon.0) 475 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-07T03:02:39.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:38 smithi145 bash[21339]: cluster 2024-03-07T03:02:37.709548+0000 mon.a (mon.0) 476 : cluster 0 osdmap e29: 5 total, 4 up, 5 in 2024-03-07T03:02:39.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:38 smithi145 bash[21339]: audit 2024-03-07T03:02:37.709818+0000 mon.a (mon.0) 477 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:39.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:38 smithi145 bash[21339]: audit 2024-03-07T03:02:37.711408+0000 mon.a (mon.0) 478 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:02:39.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:38 smithi145 bash[21339]: audit 2024-03-07T03:02:37.711421+0000 mon.b (mon.1) 13 : audit 1 from='osd.4 [v2:172.21.15.189:6800/3152992669,v1:172.21.15.189:6801/3152992669]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:02:39.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:38 smithi145 bash[30906]: cluster 2024-03-07T03:02:37.298839+0000 mgr.y (mgr.14152) 224 : cluster 0 pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:39.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:38 smithi145 bash[30906]: audit 2024-03-07T03:02:37.709247+0000 mon.a (mon.0) 475 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-07T03:02:39.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:38 smithi145 bash[30906]: cluster 2024-03-07T03:02:37.709548+0000 mon.a (mon.0) 476 : cluster 0 osdmap e29: 5 total, 4 up, 5 in 2024-03-07T03:02:39.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:38 smithi145 bash[30906]: audit 2024-03-07T03:02:37.709818+0000 mon.a (mon.0) 477 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:39.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:38 smithi145 bash[30906]: audit 2024-03-07T03:02:37.711408+0000 mon.a (mon.0) 478 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:02:39.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:38 smithi145 bash[30906]: audit 2024-03-07T03:02:37.711421+0000 mon.b (mon.1) 13 : audit 1 from='osd.4 [v2:172.21.15.189:6800/3152992669,v1:172.21.15.189:6801/3152992669]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:02:39.235 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 4 on host 'smithi189' 2024-03-07T03:02:39.777 DEBUG:teuthology.orchestra.run.smithi189:osd.4> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.4.service 2024-03-07T03:02:39.780 INFO:tasks.cephadm:Deploying osd.5 on smithi189 with /dev/vg_nvme/lv_3... 2024-03-07T03:02:39.780 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-07T03:02:40.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[21339]: audit 2024-03-07T03:02:38.715651+0000 mon.a (mon.0) 479 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-07T03:02:40.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[21339]: cluster 2024-03-07T03:02:38.715874+0000 mon.a (mon.0) 480 : cluster 0 osdmap e30: 5 total, 4 up, 5 in 2024-03-07T03:02:40.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[21339]: audit 2024-03-07T03:02:38.716567+0000 mon.a (mon.0) 481 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:40.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[21339]: audit 2024-03-07T03:02:38.727848+0000 mon.a (mon.0) 482 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:40.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[21339]: audit 2024-03-07T03:02:39.217070+0000 mon.a (mon.0) 483 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:40.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[21339]: audit 2024-03-07T03:02:39.227328+0000 mon.a (mon.0) 484 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:40.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[21339]: audit 2024-03-07T03:02:39.246787+0000 mon.a (mon.0) 485 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:40.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[30906]: audit 2024-03-07T03:02:38.715651+0000 mon.a (mon.0) 479 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-07T03:02:40.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[30906]: cluster 2024-03-07T03:02:38.715874+0000 mon.a (mon.0) 480 : cluster 0 osdmap e30: 5 total, 4 up, 5 in 2024-03-07T03:02:40.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[30906]: audit 2024-03-07T03:02:38.716567+0000 mon.a (mon.0) 481 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:40.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[30906]: audit 2024-03-07T03:02:38.727848+0000 mon.a (mon.0) 482 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:40.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[30906]: audit 2024-03-07T03:02:39.217070+0000 mon.a (mon.0) 483 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:40.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[30906]: audit 2024-03-07T03:02:39.227328+0000 mon.a (mon.0) 484 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:40.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:39 smithi145 bash[30906]: audit 2024-03-07T03:02:39.246787+0000 mon.a (mon.0) 485 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:40.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:39 smithi189 bash[23614]: audit 2024-03-07T03:02:38.715651+0000 mon.a (mon.0) 479 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-07T03:02:40.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:39 smithi189 bash[23614]: cluster 2024-03-07T03:02:38.715874+0000 mon.a (mon.0) 480 : cluster 0 osdmap e30: 5 total, 4 up, 5 in 2024-03-07T03:02:40.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:39 smithi189 bash[23614]: audit 2024-03-07T03:02:38.716567+0000 mon.a (mon.0) 481 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:40.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:39 smithi189 bash[23614]: audit 2024-03-07T03:02:38.727848+0000 mon.a (mon.0) 482 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:40.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:39 smithi189 bash[23614]: audit 2024-03-07T03:02:39.217070+0000 mon.a (mon.0) 483 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:40.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:39 smithi189 bash[23614]: audit 2024-03-07T03:02:39.227328+0000 mon.a (mon.0) 484 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:40.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:39 smithi189 bash[23614]: audit 2024-03-07T03:02:39.246787+0000 mon.a (mon.0) 485 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:41.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[30906]: cluster 2024-03-07T03:02:38.150595+0000 osd.4 (osd.4) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:02:41.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[30906]: cluster 2024-03-07T03:02:38.150716+0000 osd.4 (osd.4) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:02:41.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[30906]: cluster 2024-03-07T03:02:39.299304+0000 mgr.y (mgr.14152) 225 : cluster 0 pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:41.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[30906]: audit 2024-03-07T03:02:39.722455+0000 mon.a (mon.0) 486 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:41.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[30906]: cluster 2024-03-07T03:02:39.733247+0000 mon.a (mon.0) 487 : cluster 1 osd.4 [v2:172.21.15.189:6800/3152992669,v1:172.21.15.189:6801/3152992669] boot 2024-03-07T03:02:41.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[30906]: cluster 2024-03-07T03:02:39.733449+0000 mon.a (mon.0) 488 : cluster 0 osdmap e31: 5 total, 5 up, 5 in 2024-03-07T03:02:41.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[30906]: audit 2024-03-07T03:02:39.735164+0000 mon.a (mon.0) 489 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:41.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[30906]: cluster 2024-03-07T03:02:40.734578+0000 mon.a (mon.0) 490 : cluster 0 osdmap e32: 5 total, 5 up, 5 in 2024-03-07T03:02:41.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[21339]: cluster 2024-03-07T03:02:38.150595+0000 osd.4 (osd.4) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:02:41.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[21339]: cluster 2024-03-07T03:02:38.150716+0000 osd.4 (osd.4) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:02:41.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[21339]: cluster 2024-03-07T03:02:39.299304+0000 mgr.y (mgr.14152) 225 : cluster 0 pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:41.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[21339]: audit 2024-03-07T03:02:39.722455+0000 mon.a (mon.0) 486 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:41.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[21339]: cluster 2024-03-07T03:02:39.733247+0000 mon.a (mon.0) 487 : cluster 1 osd.4 [v2:172.21.15.189:6800/3152992669,v1:172.21.15.189:6801/3152992669] boot 2024-03-07T03:02:41.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[21339]: cluster 2024-03-07T03:02:39.733449+0000 mon.a (mon.0) 488 : cluster 0 osdmap e31: 5 total, 5 up, 5 in 2024-03-07T03:02:41.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[21339]: audit 2024-03-07T03:02:39.735164+0000 mon.a (mon.0) 489 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:41.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:40 smithi145 bash[21339]: cluster 2024-03-07T03:02:40.734578+0000 mon.a (mon.0) 490 : cluster 0 osdmap e32: 5 total, 5 up, 5 in 2024-03-07T03:02:41.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:40 smithi189 bash[23614]: cluster 2024-03-07T03:02:38.150595+0000 osd.4 (osd.4) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:02:41.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:40 smithi189 bash[23614]: cluster 2024-03-07T03:02:38.150716+0000 osd.4 (osd.4) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:02:41.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:40 smithi189 bash[23614]: cluster 2024-03-07T03:02:39.299304+0000 mgr.y (mgr.14152) 225 : cluster 0 pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:02:41.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:40 smithi189 bash[23614]: audit 2024-03-07T03:02:39.722455+0000 mon.a (mon.0) 486 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:41.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:40 smithi189 bash[23614]: cluster 2024-03-07T03:02:39.733247+0000 mon.a (mon.0) 487 : cluster 1 osd.4 [v2:172.21.15.189:6800/3152992669,v1:172.21.15.189:6801/3152992669] boot 2024-03-07T03:02:41.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:40 smithi189 bash[23614]: cluster 2024-03-07T03:02:39.733449+0000 mon.a (mon.0) 488 : cluster 0 osdmap e31: 5 total, 5 up, 5 in 2024-03-07T03:02:41.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:40 smithi189 bash[23614]: audit 2024-03-07T03:02:39.735164+0000 mon.a (mon.0) 489 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:02:41.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:40 smithi189 bash[23614]: cluster 2024-03-07T03:02:40.734578+0000 mon.a (mon.0) 490 : cluster 0 osdmap e32: 5 total, 5 up, 5 in 2024-03-07T03:02:43.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:42 smithi145 bash[21339]: cluster 2024-03-07T03:02:41.299900+0000 mgr.y (mgr.14152) 226 : cluster 0 pgmap v199: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:02:43.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:42 smithi145 bash[21339]: cluster 2024-03-07T03:02:41.735439+0000 mon.a (mon.0) 491 : cluster 0 osdmap e33: 5 total, 5 up, 5 in 2024-03-07T03:02:43.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:42 smithi145 bash[30906]: cluster 2024-03-07T03:02:41.299900+0000 mgr.y (mgr.14152) 226 : cluster 0 pgmap v199: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:02:43.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:42 smithi145 bash[30906]: cluster 2024-03-07T03:02:41.735439+0000 mon.a (mon.0) 491 : cluster 0 osdmap e33: 5 total, 5 up, 5 in 2024-03-07T03:02:43.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:42 smithi189 bash[23614]: cluster 2024-03-07T03:02:41.299900+0000 mgr.y (mgr.14152) 226 : cluster 0 pgmap v199: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:02:43.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:42 smithi189 bash[23614]: cluster 2024-03-07T03:02:41.735439+0000 mon.a (mon.0) 491 : cluster 0 osdmap e33: 5 total, 5 up, 5 in 2024-03-07T03:02:44.062 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:02:44.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:43 smithi145 bash[21339]: cluster 2024-03-07T03:02:42.749522+0000 mon.a (mon.0) 492 : cluster 0 osdmap e34: 5 total, 5 up, 5 in 2024-03-07T03:02:44.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:43 smithi145 bash[30906]: cluster 2024-03-07T03:02:42.749522+0000 mon.a (mon.0) 492 : cluster 0 osdmap e34: 5 total, 5 up, 5 in 2024-03-07T03:02:44.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:43 smithi189 bash[23614]: cluster 2024-03-07T03:02:42.749522+0000 mon.a (mon.0) 492 : cluster 0 osdmap e34: 5 total, 5 up, 5 in 2024-03-07T03:02:45.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:44 smithi189 bash[23614]: cluster 2024-03-07T03:02:43.300438+0000 mgr.y (mgr.14152) 227 : cluster 0 pgmap v202: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:02:45.006 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:44 smithi189 bash[23614]: cluster 2024-03-07T03:02:43.752766+0000 mon.a (mon.0) 493 : cluster 0 osdmap e35: 5 total, 5 up, 5 in 2024-03-07T03:02:45.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:44 smithi145 bash[30906]: cluster 2024-03-07T03:02:43.300438+0000 mgr.y (mgr.14152) 227 : cluster 0 pgmap v202: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:02:45.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:44 smithi145 bash[30906]: cluster 2024-03-07T03:02:43.752766+0000 mon.a (mon.0) 493 : cluster 0 osdmap e35: 5 total, 5 up, 5 in 2024-03-07T03:02:45.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:44 smithi145 bash[21339]: cluster 2024-03-07T03:02:43.300438+0000 mgr.y (mgr.14152) 227 : cluster 0 pgmap v202: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:02:45.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:44 smithi145 bash[21339]: cluster 2024-03-07T03:02:43.752766+0000 mon.a (mon.0) 493 : cluster 0 osdmap e35: 5 total, 5 up, 5 in 2024-03-07T03:02:47.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:46 smithi145 bash[30906]: cluster 2024-03-07T03:02:45.301039+0000 mgr.y (mgr.14152) 228 : cluster 0 pgmap v204: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:02:47.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:46 smithi145 bash[21339]: cluster 2024-03-07T03:02:45.301039+0000 mgr.y (mgr.14152) 228 : cluster 0 pgmap v204: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:02:47.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:46 smithi189 bash[23614]: cluster 2024-03-07T03:02:45.301039+0000 mgr.y (mgr.14152) 228 : cluster 0 pgmap v204: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:02:47.611 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-07T03:02:47.611 INFO:teuthology.orchestra.run.smithi189.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-07T03:02:47.611 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10+0 records in 2024-03-07T03:02:47.612 INFO:teuthology.orchestra.run.smithi189.stderr:10+0 records out 2024-03-07T03:02:47.612 INFO:teuthology.orchestra.run.smithi189.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0141511 s, 741 MB/s 2024-03-07T03:02:47.612 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping successful for: 2024-03-07T03:02:48.273 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi189:vg_nvme/lv_3 2024-03-07T03:02:49.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[21339]: cluster 2024-03-07T03:02:47.301637+0000 mgr.y (mgr.14152) 229 : cluster 0 pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 136 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-07T03:02:49.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[21339]: audit 2024-03-07T03:02:48.649505+0000 mon.a (mon.0) 494 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:49.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[21339]: audit 2024-03-07T03:02:48.658662+0000 mon.a (mon.0) 495 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:49.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[21339]: audit 2024-03-07T03:02:48.660751+0000 mon.a (mon.0) 496 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:49.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[21339]: audit 2024-03-07T03:02:48.670548+0000 mon.a (mon.0) 497 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:49.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[21339]: audit 2024-03-07T03:02:48.672749+0000 mon.a (mon.0) 498 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:49.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[21339]: audit 2024-03-07T03:02:48.674506+0000 mon.a (mon.0) 499 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:49.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[21339]: audit 2024-03-07T03:02:48.684644+0000 mon.a (mon.0) 500 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:49.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[30906]: cluster 2024-03-07T03:02:47.301637+0000 mgr.y (mgr.14152) 229 : cluster 0 pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 136 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-07T03:02:49.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[30906]: audit 2024-03-07T03:02:48.649505+0000 mon.a (mon.0) 494 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:49.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[30906]: audit 2024-03-07T03:02:48.658662+0000 mon.a (mon.0) 495 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:49.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[30906]: audit 2024-03-07T03:02:48.660751+0000 mon.a (mon.0) 496 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:49.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[30906]: audit 2024-03-07T03:02:48.670548+0000 mon.a (mon.0) 497 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:49.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[30906]: audit 2024-03-07T03:02:48.672749+0000 mon.a (mon.0) 498 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:49.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[30906]: audit 2024-03-07T03:02:48.674506+0000 mon.a (mon.0) 499 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:49.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:49 smithi145 bash[30906]: audit 2024-03-07T03:02:48.684644+0000 mon.a (mon.0) 500 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:49.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:49 smithi189 bash[23614]: cluster 2024-03-07T03:02:47.301637+0000 mgr.y (mgr.14152) 229 : cluster 0 pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 136 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-07T03:02:49.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:49 smithi189 bash[23614]: audit 2024-03-07T03:02:48.649505+0000 mon.a (mon.0) 494 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:49.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:49 smithi189 bash[23614]: audit 2024-03-07T03:02:48.658662+0000 mon.a (mon.0) 495 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:49.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:49 smithi189 bash[23614]: audit 2024-03-07T03:02:48.660751+0000 mon.a (mon.0) 496 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:49 smithi189 bash[23614]: audit 2024-03-07T03:02:48.670548+0000 mon.a (mon.0) 497 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:49 smithi189 bash[23614]: audit 2024-03-07T03:02:48.672749+0000 mon.a (mon.0) 498 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:49 smithi189 bash[23614]: audit 2024-03-07T03:02:48.674506+0000 mon.a (mon.0) 499 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:49 smithi189 bash[23614]: audit 2024-03-07T03:02:48.684644+0000 mon.a (mon.0) 500 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:02:50.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:50 smithi145 bash[21339]: cephadm 2024-03-07T03:02:48.640328+0000 mgr.y (mgr.14152) 230 : cephadm 1 Detected new or changed devices on smithi189 2024-03-07T03:02:50.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:50 smithi145 bash[21339]: cephadm 2024-03-07T03:02:48.662166+0000 mgr.y (mgr.14152) 231 : cephadm 1 Adjusting osd_memory_target on smithi189 to 17265M 2024-03-07T03:02:50.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:50 smithi145 bash[30906]: cephadm 2024-03-07T03:02:48.640328+0000 mgr.y (mgr.14152) 230 : cephadm 1 Detected new or changed devices on smithi189 2024-03-07T03:02:50.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:50 smithi145 bash[30906]: cephadm 2024-03-07T03:02:48.662166+0000 mgr.y (mgr.14152) 231 : cephadm 1 Adjusting osd_memory_target on smithi189 to 17265M 2024-03-07T03:02:50.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:50 smithi189 bash[23614]: cephadm 2024-03-07T03:02:48.640328+0000 mgr.y (mgr.14152) 230 : cephadm 1 Detected new or changed devices on smithi189 2024-03-07T03:02:50.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:50 smithi189 bash[23614]: cephadm 2024-03-07T03:02:48.662166+0000 mgr.y (mgr.14152) 231 : cephadm 1 Adjusting osd_memory_target on smithi189 to 17265M 2024-03-07T03:02:51.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:51 smithi145 bash[21339]: cluster 2024-03-07T03:02:49.302250+0000 mgr.y (mgr.14152) 232 : cluster 0 pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-03-07T03:02:51.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:51 smithi145 bash[30906]: cluster 2024-03-07T03:02:49.302250+0000 mgr.y (mgr.14152) 232 : cluster 0 pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-03-07T03:02:51.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:51 smithi189 bash[23614]: cluster 2024-03-07T03:02:49.302250+0000 mgr.y (mgr.14152) 232 : cluster 0 pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-03-07T03:02:52.947 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:02:53.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:53 smithi145 bash[21339]: cluster 2024-03-07T03:02:51.302822+0000 mgr.y (mgr.14152) 233 : cluster 0 pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-03-07T03:02:53.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:53 smithi145 bash[30906]: cluster 2024-03-07T03:02:51.302822+0000 mgr.y (mgr.14152) 233 : cluster 0 pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-03-07T03:02:53.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:53 smithi189 bash[23614]: cluster 2024-03-07T03:02:51.302822+0000 mgr.y (mgr.14152) 233 : cluster 0 pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-03-07T03:02:55.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:55 smithi145 bash[21339]: cluster 2024-03-07T03:02:53.303392+0000 mgr.y (mgr.14152) 234 : cluster 0 pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:02:55.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:55 smithi145 bash[30906]: cluster 2024-03-07T03:02:53.303392+0000 mgr.y (mgr.14152) 234 : cluster 0 pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:02:55.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:55 smithi189 bash[23614]: cluster 2024-03-07T03:02:53.303392+0000 mgr.y (mgr.14152) 234 : cluster 0 pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:02:56.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:56 smithi145 bash[21339]: audit 2024-03-07T03:02:55.391619+0000 mon.a (mon.0) 501 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:02:56.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:56 smithi145 bash[21339]: audit 2024-03-07T03:02:55.398271+0000 mon.a (mon.0) 502 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:02:56.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:56 smithi145 bash[21339]: audit 2024-03-07T03:02:55.400239+0000 mon.a (mon.0) 503 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:56.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:56 smithi145 bash[30906]: audit 2024-03-07T03:02:55.391619+0000 mon.a (mon.0) 501 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:02:56.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:56 smithi145 bash[30906]: audit 2024-03-07T03:02:55.398271+0000 mon.a (mon.0) 502 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:02:56.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:56 smithi145 bash[30906]: audit 2024-03-07T03:02:55.400239+0000 mon.a (mon.0) 503 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:56.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:56 smithi189 bash[23614]: audit 2024-03-07T03:02:55.391619+0000 mon.a (mon.0) 501 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:02:56.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:56 smithi189 bash[23614]: audit 2024-03-07T03:02:55.398271+0000 mon.a (mon.0) 502 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:02:56.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:56 smithi189 bash[23614]: audit 2024-03-07T03:02:55.400239+0000 mon.a (mon.0) 503 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:57.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:57 smithi145 bash[21339]: cluster 2024-03-07T03:02:55.303970+0000 mgr.y (mgr.14152) 235 : cluster 0 pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-03-07T03:02:57.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:57 smithi145 bash[21339]: audit 2024-03-07T03:02:55.384665+0000 mgr.y (mgr.14152) 236 : audit 0 from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:02:57.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:57 smithi145 bash[30906]: cluster 2024-03-07T03:02:55.303970+0000 mgr.y (mgr.14152) 235 : cluster 0 pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-03-07T03:02:57.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:57 smithi145 bash[30906]: audit 2024-03-07T03:02:55.384665+0000 mgr.y (mgr.14152) 236 : audit 0 from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:02:57.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:57 smithi189 bash[23614]: cluster 2024-03-07T03:02:55.303970+0000 mgr.y (mgr.14152) 235 : cluster 0 pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-03-07T03:02:57.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:57 smithi189 bash[23614]: audit 2024-03-07T03:02:55.384665+0000 mgr.y (mgr.14152) 236 : audit 0 from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:02:59.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:02:59 smithi145 bash[21339]: cluster 2024-03-07T03:02:57.304556+0000 mgr.y (mgr.14152) 237 : cluster 0 pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:02:59.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:02:59 smithi145 bash[30906]: cluster 2024-03-07T03:02:57.304556+0000 mgr.y (mgr.14152) 237 : cluster 0 pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:02:59.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:02:59 smithi189 bash[23614]: cluster 2024-03-07T03:02:57.304556+0000 mgr.y (mgr.14152) 237 : cluster 0 pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:03:01.567 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:01 smithi189 bash[23614]: cluster 2024-03-07T03:02:59.305094+0000 mgr.y (mgr.14152) 238 : cluster 0 pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:01.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:01 smithi145 bash[21339]: cluster 2024-03-07T03:02:59.305094+0000 mgr.y (mgr.14152) 238 : cluster 0 pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:01.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:01 smithi145 bash[30906]: cluster 2024-03-07T03:02:59.305094+0000 mgr.y (mgr.14152) 238 : cluster 0 pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:03.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:03 smithi145 bash[21339]: cluster 2024-03-07T03:03:01.305655+0000 mgr.y (mgr.14152) 239 : cluster 0 pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:03.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:03 smithi145 bash[21339]: audit 2024-03-07T03:03:03.119333+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.189:0/337691950' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "531d5127-60fe-4669-ab7d-fe4f5e5753cf"}]: dispatch 2024-03-07T03:03:03.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:03 smithi145 bash[21339]: audit 2024-03-07T03:03:03.119659+0000 mon.a (mon.0) 504 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "531d5127-60fe-4669-ab7d-fe4f5e5753cf"}]: dispatch 2024-03-07T03:03:03.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:03 smithi145 bash[21339]: audit 2024-03-07T03:03:03.129038+0000 mon.a (mon.0) 505 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "531d5127-60fe-4669-ab7d-fe4f5e5753cf"}]': finished 2024-03-07T03:03:03.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:03 smithi145 bash[21339]: cluster 2024-03-07T03:03:03.129223+0000 mon.a (mon.0) 506 : cluster 0 osdmap e36: 6 total, 5 up, 6 in 2024-03-07T03:03:03.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:03 smithi145 bash[21339]: audit 2024-03-07T03:03:03.129501+0000 mon.a (mon.0) 507 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:03.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:03 smithi145 bash[30906]: cluster 2024-03-07T03:03:01.305655+0000 mgr.y (mgr.14152) 239 : cluster 0 pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:03.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:03 smithi145 bash[30906]: audit 2024-03-07T03:03:03.119333+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.189:0/337691950' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "531d5127-60fe-4669-ab7d-fe4f5e5753cf"}]: dispatch 2024-03-07T03:03:03.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:03 smithi145 bash[30906]: audit 2024-03-07T03:03:03.119659+0000 mon.a (mon.0) 504 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "531d5127-60fe-4669-ab7d-fe4f5e5753cf"}]: dispatch 2024-03-07T03:03:03.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:03 smithi145 bash[30906]: audit 2024-03-07T03:03:03.129038+0000 mon.a (mon.0) 505 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "531d5127-60fe-4669-ab7d-fe4f5e5753cf"}]': finished 2024-03-07T03:03:03.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:03 smithi145 bash[30906]: cluster 2024-03-07T03:03:03.129223+0000 mon.a (mon.0) 506 : cluster 0 osdmap e36: 6 total, 5 up, 6 in 2024-03-07T03:03:03.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:03 smithi145 bash[30906]: audit 2024-03-07T03:03:03.129501+0000 mon.a (mon.0) 507 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:03.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:03 smithi189 bash[23614]: cluster 2024-03-07T03:03:01.305655+0000 mgr.y (mgr.14152) 239 : cluster 0 pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:03.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:03 smithi189 bash[23614]: audit 2024-03-07T03:03:03.119333+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.189:0/337691950' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "531d5127-60fe-4669-ab7d-fe4f5e5753cf"}]: dispatch 2024-03-07T03:03:03.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:03 smithi189 bash[23614]: audit 2024-03-07T03:03:03.119659+0000 mon.a (mon.0) 504 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "531d5127-60fe-4669-ab7d-fe4f5e5753cf"}]: dispatch 2024-03-07T03:03:03.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:03 smithi189 bash[23614]: audit 2024-03-07T03:03:03.129038+0000 mon.a (mon.0) 505 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "531d5127-60fe-4669-ab7d-fe4f5e5753cf"}]': finished 2024-03-07T03:03:03.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:03 smithi189 bash[23614]: cluster 2024-03-07T03:03:03.129223+0000 mon.a (mon.0) 506 : cluster 0 osdmap e36: 6 total, 5 up, 6 in 2024-03-07T03:03:03.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:03 smithi189 bash[23614]: audit 2024-03-07T03:03:03.129501+0000 mon.a (mon.0) 507 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:04.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:04 smithi145 bash[21339]: cluster 2024-03-07T03:03:03.306205+0000 mgr.y (mgr.14152) 240 : cluster 0 pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:04.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:04 smithi145 bash[21339]: audit 2024-03-07T03:03:03.826743+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.189:0/1001779931' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:03:04.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:04 smithi145 bash[30906]: cluster 2024-03-07T03:03:03.306205+0000 mgr.y (mgr.14152) 240 : cluster 0 pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:04.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:04 smithi145 bash[30906]: audit 2024-03-07T03:03:03.826743+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.189:0/1001779931' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:03:04.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:04 smithi189 bash[23614]: cluster 2024-03-07T03:03:03.306205+0000 mgr.y (mgr.14152) 240 : cluster 0 pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:04.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:04 smithi189 bash[23614]: audit 2024-03-07T03:03:03.826743+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.189:0/1001779931' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:03:06.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:06 smithi189 bash[23614]: cluster 2024-03-07T03:03:05.306744+0000 mgr.y (mgr.14152) 241 : cluster 0 pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:06.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:06 smithi145 bash[21339]: cluster 2024-03-07T03:03:05.306744+0000 mgr.y (mgr.14152) 241 : cluster 0 pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:06.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:06 smithi145 bash[30906]: cluster 2024-03-07T03:03:05.306744+0000 mgr.y (mgr.14152) 241 : cluster 0 pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:09.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:09 smithi145 bash[21339]: cluster 2024-03-07T03:03:07.307283+0000 mgr.y (mgr.14152) 242 : cluster 0 pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:09.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:09 smithi145 bash[30906]: cluster 2024-03-07T03:03:07.307283+0000 mgr.y (mgr.14152) 242 : cluster 0 pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:09.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:09 smithi189 bash[23614]: cluster 2024-03-07T03:03:07.307283+0000 mgr.y (mgr.14152) 242 : cluster 0 pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:11.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:11 smithi145 bash[21339]: cluster 2024-03-07T03:03:09.307828+0000 mgr.y (mgr.14152) 243 : cluster 0 pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:11.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:11 smithi145 bash[30906]: cluster 2024-03-07T03:03:09.307828+0000 mgr.y (mgr.14152) 243 : cluster 0 pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:11.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:11 smithi189 bash[23614]: cluster 2024-03-07T03:03:09.307828+0000 mgr.y (mgr.14152) 243 : cluster 0 pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:13.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:13 smithi145 bash[30906]: cluster 2024-03-07T03:03:11.308471+0000 mgr.y (mgr.14152) 244 : cluster 0 pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:13.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:13 smithi145 bash[21339]: cluster 2024-03-07T03:03:11.308471+0000 mgr.y (mgr.14152) 244 : cluster 0 pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:13.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:13 smithi189 bash[23614]: cluster 2024-03-07T03:03:11.308471+0000 mgr.y (mgr.14152) 244 : cluster 0 pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:15.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:15 smithi189 bash[23614]: cluster 2024-03-07T03:03:13.309143+0000 mgr.y (mgr.14152) 245 : cluster 0 pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:15.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:15 smithi189 bash[23614]: audit 2024-03-07T03:03:14.780485+0000 mon.a (mon.0) 508 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-07T03:03:15.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:15 smithi189 bash[23614]: audit 2024-03-07T03:03:14.782245+0000 mon.a (mon.0) 509 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:15.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:15 smithi145 bash[30906]: cluster 2024-03-07T03:03:13.309143+0000 mgr.y (mgr.14152) 245 : cluster 0 pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:15.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:15 smithi145 bash[30906]: audit 2024-03-07T03:03:14.780485+0000 mon.a (mon.0) 508 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-07T03:03:15.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:15 smithi145 bash[30906]: audit 2024-03-07T03:03:14.782245+0000 mon.a (mon.0) 509 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:15.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:15 smithi145 bash[21339]: cluster 2024-03-07T03:03:13.309143+0000 mgr.y (mgr.14152) 245 : cluster 0 pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:15.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:15 smithi145 bash[21339]: audit 2024-03-07T03:03:14.780485+0000 mon.a (mon.0) 508 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-07T03:03:15.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:15 smithi145 bash[21339]: audit 2024-03-07T03:03:14.782245+0000 mon.a (mon.0) 509 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:16.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:16 smithi145 bash[21339]: cephadm 2024-03-07T03:03:14.783774+0000 mgr.y (mgr.14152) 246 : cephadm 1 Deploying daemon osd.5 on smithi189 2024-03-07T03:03:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:16 smithi145 bash[30906]: cephadm 2024-03-07T03:03:14.783774+0000 mgr.y (mgr.14152) 246 : cephadm 1 Deploying daemon osd.5 on smithi189 2024-03-07T03:03:16.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:16 smithi189 bash[23614]: cephadm 2024-03-07T03:03:14.783774+0000 mgr.y (mgr.14152) 246 : cephadm 1 Deploying daemon osd.5 on smithi189 2024-03-07T03:03:17.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:17 smithi145 bash[30906]: cluster 2024-03-07T03:03:15.309806+0000 mgr.y (mgr.14152) 247 : cluster 0 pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:17.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:17 smithi145 bash[21339]: cluster 2024-03-07T03:03:15.309806+0000 mgr.y (mgr.14152) 247 : cluster 0 pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:17.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:17 smithi189 bash[23614]: cluster 2024-03-07T03:03:15.309806+0000 mgr.y (mgr.14152) 247 : cluster 0 pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:18.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:03:18 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:18.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:18 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:18.582 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:03:18 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:18.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:18 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:18.838 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:03:18 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:18.838 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:03:18 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:19.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:19 smithi145 bash[21339]: cluster 2024-03-07T03:03:17.310381+0000 mgr.y (mgr.14152) 248 : cluster 0 pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:19.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:19 smithi145 bash[21339]: audit 2024-03-07T03:03:18.854235+0000 mon.a (mon.0) 510 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:03:19.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:19 smithi145 bash[21339]: audit 2024-03-07T03:03:18.860310+0000 mon.a (mon.0) 511 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:19.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:19 smithi145 bash[21339]: audit 2024-03-07T03:03:18.872085+0000 mon.a (mon.0) 512 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:19.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:19 smithi145 bash[30906]: cluster 2024-03-07T03:03:17.310381+0000 mgr.y (mgr.14152) 248 : cluster 0 pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:19.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:19 smithi145 bash[30906]: audit 2024-03-07T03:03:18.854235+0000 mon.a (mon.0) 510 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:03:19.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:19 smithi145 bash[30906]: audit 2024-03-07T03:03:18.860310+0000 mon.a (mon.0) 511 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:19.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:19 smithi145 bash[30906]: audit 2024-03-07T03:03:18.872085+0000 mon.a (mon.0) 512 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:19.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:19 smithi189 bash[23614]: cluster 2024-03-07T03:03:17.310381+0000 mgr.y (mgr.14152) 248 : cluster 0 pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:19.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:19 smithi189 bash[23614]: audit 2024-03-07T03:03:18.854235+0000 mon.a (mon.0) 510 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:03:19.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:19 smithi189 bash[23614]: audit 2024-03-07T03:03:18.860310+0000 mon.a (mon.0) 511 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:19.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:19 smithi189 bash[23614]: audit 2024-03-07T03:03:18.872085+0000 mon.a (mon.0) 512 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:21.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:21 smithi145 bash[21339]: cluster 2024-03-07T03:03:19.310865+0000 mgr.y (mgr.14152) 249 : cluster 0 pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:21.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:21 smithi145 bash[30906]: cluster 2024-03-07T03:03:19.310865+0000 mgr.y (mgr.14152) 249 : cluster 0 pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:21.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:21 smithi189 bash[23614]: cluster 2024-03-07T03:03:19.310865+0000 mgr.y (mgr.14152) 249 : cluster 0 pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:23.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:23 smithi145 bash[21339]: cluster 2024-03-07T03:03:21.311369+0000 mgr.y (mgr.14152) 250 : cluster 0 pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:23.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:23 smithi145 bash[30906]: cluster 2024-03-07T03:03:21.311369+0000 mgr.y (mgr.14152) 250 : cluster 0 pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:23.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:23 smithi189 bash[23614]: cluster 2024-03-07T03:03:21.311369+0000 mgr.y (mgr.14152) 250 : cluster 0 pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:25.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[21339]: cluster 2024-03-07T03:03:23.311913+0000 mgr.y (mgr.14152) 251 : cluster 0 pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:25.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[21339]: audit 2024-03-07T03:03:24.056106+0000 mon.a (mon.0) 513 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:25.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[21339]: audit 2024-03-07T03:03:24.065950+0000 mon.a (mon.0) 514 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:25.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[21339]: audit 2024-03-07T03:03:24.068054+0000 mon.a (mon.0) 515 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:25.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[21339]: audit 2024-03-07T03:03:24.069827+0000 mon.a (mon.0) 516 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:25.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[21339]: audit 2024-03-07T03:03:24.080162+0000 mon.a (mon.0) 517 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:25.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[21339]: audit 2024-03-07T03:03:25.010382+0000 mon.b (mon.1) 16 : audit 1 from='osd.5 [v2:172.21.15.189:6808/2096444201,v1:172.21.15.189:6809/2096444201]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-07T03:03:25.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[21339]: audit 2024-03-07T03:03:25.010598+0000 mon.a (mon.0) 518 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-07T03:03:25.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[30906]: cluster 2024-03-07T03:03:23.311913+0000 mgr.y (mgr.14152) 251 : cluster 0 pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:25.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[30906]: audit 2024-03-07T03:03:24.056106+0000 mon.a (mon.0) 513 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:25.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[30906]: audit 2024-03-07T03:03:24.065950+0000 mon.a (mon.0) 514 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:25.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[30906]: audit 2024-03-07T03:03:24.068054+0000 mon.a (mon.0) 515 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:25.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[30906]: audit 2024-03-07T03:03:24.069827+0000 mon.a (mon.0) 516 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:25.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[30906]: audit 2024-03-07T03:03:24.080162+0000 mon.a (mon.0) 517 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:25.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[30906]: audit 2024-03-07T03:03:25.010382+0000 mon.b (mon.1) 16 : audit 1 from='osd.5 [v2:172.21.15.189:6808/2096444201,v1:172.21.15.189:6809/2096444201]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-07T03:03:25.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:25 smithi145 bash[30906]: audit 2024-03-07T03:03:25.010598+0000 mon.a (mon.0) 518 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-07T03:03:25.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:25 smithi189 bash[23614]: cluster 2024-03-07T03:03:23.311913+0000 mgr.y (mgr.14152) 251 : cluster 0 pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:25.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:25 smithi189 bash[23614]: audit 2024-03-07T03:03:24.056106+0000 mon.a (mon.0) 513 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:25.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:25 smithi189 bash[23614]: audit 2024-03-07T03:03:24.065950+0000 mon.a (mon.0) 514 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:25.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:25 smithi189 bash[23614]: audit 2024-03-07T03:03:24.068054+0000 mon.a (mon.0) 515 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:25.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:25 smithi189 bash[23614]: audit 2024-03-07T03:03:24.069827+0000 mon.a (mon.0) 516 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:25.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:25 smithi189 bash[23614]: audit 2024-03-07T03:03:24.080162+0000 mon.a (mon.0) 517 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:25.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:25 smithi189 bash[23614]: audit 2024-03-07T03:03:25.010382+0000 mon.b (mon.1) 16 : audit 1 from='osd.5 [v2:172.21.15.189:6808/2096444201,v1:172.21.15.189:6809/2096444201]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-07T03:03:25.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:25 smithi189 bash[23614]: audit 2024-03-07T03:03:25.010598+0000 mon.a (mon.0) 518 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-07T03:03:26.556 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:26 smithi189 bash[23614]: audit 2024-03-07T03:03:25.296147+0000 mon.a (mon.0) 519 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-07T03:03:26.556 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:26 smithi189 bash[23614]: cluster 2024-03-07T03:03:25.296273+0000 mon.a (mon.0) 520 : cluster 0 osdmap e37: 6 total, 5 up, 6 in 2024-03-07T03:03:26.556 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:26 smithi189 bash[23614]: audit 2024-03-07T03:03:25.296589+0000 mon.a (mon.0) 521 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:26.556 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:26 smithi189 bash[23614]: audit 2024-03-07T03:03:25.298258+0000 mon.b (mon.1) 17 : audit 1 from='osd.5 [v2:172.21.15.189:6808/2096444201,v1:172.21.15.189:6809/2096444201]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:03:26.556 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:26 smithi189 bash[23614]: audit 2024-03-07T03:03:25.298503+0000 mon.a (mon.0) 522 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:03:26.556 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:26 smithi189 bash[23614]: cluster 2024-03-07T03:03:25.312433+0000 mgr.y (mgr.14152) 252 : cluster 0 pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:26.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:26 smithi145 bash[21339]: audit 2024-03-07T03:03:25.296147+0000 mon.a (mon.0) 519 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-07T03:03:26.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:26 smithi145 bash[21339]: cluster 2024-03-07T03:03:25.296273+0000 mon.a (mon.0) 520 : cluster 0 osdmap e37: 6 total, 5 up, 6 in 2024-03-07T03:03:26.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:26 smithi145 bash[21339]: audit 2024-03-07T03:03:25.296589+0000 mon.a (mon.0) 521 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:26.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:26 smithi145 bash[21339]: audit 2024-03-07T03:03:25.298258+0000 mon.b (mon.1) 17 : audit 1 from='osd.5 [v2:172.21.15.189:6808/2096444201,v1:172.21.15.189:6809/2096444201]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:03:26.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:26 smithi145 bash[21339]: audit 2024-03-07T03:03:25.298503+0000 mon.a (mon.0) 522 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:03:26.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:26 smithi145 bash[21339]: cluster 2024-03-07T03:03:25.312433+0000 mgr.y (mgr.14152) 252 : cluster 0 pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:26.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:26 smithi145 bash[30906]: audit 2024-03-07T03:03:25.296147+0000 mon.a (mon.0) 519 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-07T03:03:26.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:26 smithi145 bash[30906]: cluster 2024-03-07T03:03:25.296273+0000 mon.a (mon.0) 520 : cluster 0 osdmap e37: 6 total, 5 up, 6 in 2024-03-07T03:03:26.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:26 smithi145 bash[30906]: audit 2024-03-07T03:03:25.296589+0000 mon.a (mon.0) 521 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:26.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:26 smithi145 bash[30906]: audit 2024-03-07T03:03:25.298258+0000 mon.b (mon.1) 17 : audit 1 from='osd.5 [v2:172.21.15.189:6808/2096444201,v1:172.21.15.189:6809/2096444201]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:03:26.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:26 smithi145 bash[30906]: audit 2024-03-07T03:03:25.298503+0000 mon.a (mon.0) 522 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:03:26.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:26 smithi145 bash[30906]: cluster 2024-03-07T03:03:25.312433+0000 mgr.y (mgr.14152) 252 : cluster 0 pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:27.042 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 5 on host 'smithi189' 2024-03-07T03:03:27.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:27 smithi189 bash[23614]: audit 2024-03-07T03:03:26.298560+0000 mon.a (mon.0) 523 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-07T03:03:27.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:27 smithi189 bash[23614]: cluster 2024-03-07T03:03:26.298869+0000 mon.a (mon.0) 524 : cluster 0 osdmap e38: 6 total, 5 up, 6 in 2024-03-07T03:03:27.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:27 smithi189 bash[23614]: audit 2024-03-07T03:03:26.299857+0000 mon.a (mon.0) 525 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:27.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:27 smithi189 bash[23614]: audit 2024-03-07T03:03:26.308698+0000 mon.a (mon.0) 526 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:27.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:27 smithi189 bash[23614]: audit 2024-03-07T03:03:27.013676+0000 mon.a (mon.0) 527 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:27.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:27 smithi189 bash[23614]: audit 2024-03-07T03:03:27.024937+0000 mon.a (mon.0) 528 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:27.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:27 smithi189 bash[23614]: audit 2024-03-07T03:03:27.025621+0000 mon.a (mon.0) 529 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:03:27.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[21339]: audit 2024-03-07T03:03:26.298560+0000 mon.a (mon.0) 523 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-07T03:03:27.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[21339]: cluster 2024-03-07T03:03:26.298869+0000 mon.a (mon.0) 524 : cluster 0 osdmap e38: 6 total, 5 up, 6 in 2024-03-07T03:03:27.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[21339]: audit 2024-03-07T03:03:26.299857+0000 mon.a (mon.0) 525 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:27.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[21339]: audit 2024-03-07T03:03:26.308698+0000 mon.a (mon.0) 526 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:27.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[21339]: audit 2024-03-07T03:03:27.013676+0000 mon.a (mon.0) 527 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:27.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[21339]: audit 2024-03-07T03:03:27.024937+0000 mon.a (mon.0) 528 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:27.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[21339]: audit 2024-03-07T03:03:27.025621+0000 mon.a (mon.0) 529 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:03:27.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[30906]: audit 2024-03-07T03:03:26.298560+0000 mon.a (mon.0) 523 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-07T03:03:27.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[30906]: cluster 2024-03-07T03:03:26.298869+0000 mon.a (mon.0) 524 : cluster 0 osdmap e38: 6 total, 5 up, 6 in 2024-03-07T03:03:27.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[30906]: audit 2024-03-07T03:03:26.299857+0000 mon.a (mon.0) 525 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:27.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[30906]: audit 2024-03-07T03:03:26.308698+0000 mon.a (mon.0) 526 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:27.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[30906]: audit 2024-03-07T03:03:27.013676+0000 mon.a (mon.0) 527 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:27.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[30906]: audit 2024-03-07T03:03:27.024937+0000 mon.a (mon.0) 528 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:27.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:27 smithi145 bash[30906]: audit 2024-03-07T03:03:27.025621+0000 mon.a (mon.0) 529 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:03:27.599 DEBUG:teuthology.orchestra.run.smithi189:osd.5> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.5.service 2024-03-07T03:03:27.601 INFO:tasks.cephadm:Deploying osd.6 on smithi189 with /dev/vg_nvme/lv_2... 2024-03-07T03:03:27.601 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-07T03:03:28.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[21339]: cluster 2024-03-07T03:03:25.965174+0000 osd.5 (osd.5) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:03:28.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[21339]: cluster 2024-03-07T03:03:25.965236+0000 osd.5 (osd.5) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:03:28.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[21339]: cluster 2024-03-07T03:03:27.301701+0000 mon.a (mon.0) 530 : cluster 1 osd.5 [v2:172.21.15.189:6808/2096444201,v1:172.21.15.189:6809/2096444201] boot 2024-03-07T03:03:28.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[21339]: cluster 2024-03-07T03:03:27.301815+0000 mon.a (mon.0) 531 : cluster 0 osdmap e39: 6 total, 6 up, 6 in 2024-03-07T03:03:28.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[21339]: audit 2024-03-07T03:03:27.302582+0000 mon.a (mon.0) 532 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:28.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[21339]: cluster 2024-03-07T03:03:27.313050+0000 mgr.y (mgr.14152) 253 : cluster 0 pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:28.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[21339]: cluster 2024-03-07T03:03:28.258499+0000 mon.a (mon.0) 533 : cluster 0 osdmap e40: 6 total, 6 up, 6 in 2024-03-07T03:03:28.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[30906]: cluster 2024-03-07T03:03:25.965174+0000 osd.5 (osd.5) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:03:28.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[30906]: cluster 2024-03-07T03:03:25.965236+0000 osd.5 (osd.5) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:03:28.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[30906]: cluster 2024-03-07T03:03:27.301701+0000 mon.a (mon.0) 530 : cluster 1 osd.5 [v2:172.21.15.189:6808/2096444201,v1:172.21.15.189:6809/2096444201] boot 2024-03-07T03:03:28.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[30906]: cluster 2024-03-07T03:03:27.301815+0000 mon.a (mon.0) 531 : cluster 0 osdmap e39: 6 total, 6 up, 6 in 2024-03-07T03:03:28.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[30906]: audit 2024-03-07T03:03:27.302582+0000 mon.a (mon.0) 532 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:28.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[30906]: cluster 2024-03-07T03:03:27.313050+0000 mgr.y (mgr.14152) 253 : cluster 0 pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:28.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:28 smithi145 bash[30906]: cluster 2024-03-07T03:03:28.258499+0000 mon.a (mon.0) 533 : cluster 0 osdmap e40: 6 total, 6 up, 6 in 2024-03-07T03:03:28.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:28 smithi189 bash[23614]: cluster 2024-03-07T03:03:25.965174+0000 osd.5 (osd.5) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:03:28.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:28 smithi189 bash[23614]: cluster 2024-03-07T03:03:25.965236+0000 osd.5 (osd.5) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:03:28.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:28 smithi189 bash[23614]: cluster 2024-03-07T03:03:27.301701+0000 mon.a (mon.0) 530 : cluster 1 osd.5 [v2:172.21.15.189:6808/2096444201,v1:172.21.15.189:6809/2096444201] boot 2024-03-07T03:03:28.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:28 smithi189 bash[23614]: cluster 2024-03-07T03:03:27.301815+0000 mon.a (mon.0) 531 : cluster 0 osdmap e39: 6 total, 6 up, 6 in 2024-03-07T03:03:28.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:28 smithi189 bash[23614]: audit 2024-03-07T03:03:27.302582+0000 mon.a (mon.0) 532 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:03:28.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:28 smithi189 bash[23614]: cluster 2024-03-07T03:03:27.313050+0000 mgr.y (mgr.14152) 253 : cluster 0 pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:03:28.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:28 smithi189 bash[23614]: cluster 2024-03-07T03:03:28.258499+0000 mon.a (mon.0) 533 : cluster 0 osdmap e40: 6 total, 6 up, 6 in 2024-03-07T03:03:30.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:30 smithi145 bash[21339]: cluster 2024-03-07T03:03:29.261301+0000 mon.a (mon.0) 534 : cluster 0 osdmap e41: 6 total, 6 up, 6 in 2024-03-07T03:03:30.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:30 smithi145 bash[30906]: cluster 2024-03-07T03:03:29.261301+0000 mon.a (mon.0) 534 : cluster 0 osdmap e41: 6 total, 6 up, 6 in 2024-03-07T03:03:30.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:30 smithi189 bash[23614]: cluster 2024-03-07T03:03:29.261301+0000 mon.a (mon.0) 534 : cluster 0 osdmap e41: 6 total, 6 up, 6 in 2024-03-07T03:03:31.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:31 smithi145 bash[21339]: cluster 2024-03-07T03:03:29.313688+0000 mgr.y (mgr.14152) 254 : cluster 0 pgmap v232: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:31.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:31 smithi145 bash[21339]: cluster 2024-03-07T03:03:30.253195+0000 mon.a (mon.0) 535 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:03:31.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:31 smithi145 bash[21339]: cluster 2024-03-07T03:03:30.273852+0000 mon.a (mon.0) 536 : cluster 0 osdmap e42: 6 total, 6 up, 6 in 2024-03-07T03:03:31.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:31 smithi145 bash[30906]: cluster 2024-03-07T03:03:29.313688+0000 mgr.y (mgr.14152) 254 : cluster 0 pgmap v232: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:31.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:31 smithi145 bash[30906]: cluster 2024-03-07T03:03:30.253195+0000 mon.a (mon.0) 535 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:03:31.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:31 smithi145 bash[30906]: cluster 2024-03-07T03:03:30.273852+0000 mon.a (mon.0) 536 : cluster 0 osdmap e42: 6 total, 6 up, 6 in 2024-03-07T03:03:31.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:31 smithi189 bash[23614]: cluster 2024-03-07T03:03:29.313688+0000 mgr.y (mgr.14152) 254 : cluster 0 pgmap v232: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:31.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:31 smithi189 bash[23614]: cluster 2024-03-07T03:03:30.253195+0000 mon.a (mon.0) 535 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:03:31.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:31 smithi189 bash[23614]: cluster 2024-03-07T03:03:30.273852+0000 mon.a (mon.0) 536 : cluster 0 osdmap e42: 6 total, 6 up, 6 in 2024-03-07T03:03:31.858 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:03:32.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:32 smithi145 bash[21339]: cluster 2024-03-07T03:03:31.273325+0000 mon.a (mon.0) 537 : cluster 0 osdmap e43: 6 total, 6 up, 6 in 2024-03-07T03:03:32.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:32 smithi145 bash[30906]: cluster 2024-03-07T03:03:31.273325+0000 mon.a (mon.0) 537 : cluster 0 osdmap e43: 6 total, 6 up, 6 in 2024-03-07T03:03:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:32 smithi189 bash[23614]: cluster 2024-03-07T03:03:31.273325+0000 mon.a (mon.0) 537 : cluster 0 osdmap e43: 6 total, 6 up, 6 in 2024-03-07T03:03:33.431 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:33 smithi189 bash[23614]: cluster 2024-03-07T03:03:31.314436+0000 mgr.y (mgr.14152) 255 : cluster 0 pgmap v235: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:33.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:33 smithi145 bash[21339]: cluster 2024-03-07T03:03:31.314436+0000 mgr.y (mgr.14152) 255 : cluster 0 pgmap v235: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:33.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:33 smithi145 bash[30906]: cluster 2024-03-07T03:03:31.314436+0000 mgr.y (mgr.14152) 255 : cluster 0 pgmap v235: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:34.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:34 smithi145 bash[21339]: cluster 2024-03-07T03:03:33.315017+0000 mgr.y (mgr.14152) 256 : cluster 0 pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:34.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:34 smithi145 bash[30906]: cluster 2024-03-07T03:03:33.315017+0000 mgr.y (mgr.14152) 256 : cluster 0 pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:34.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:34 smithi189 bash[23614]: cluster 2024-03-07T03:03:33.315017+0000 mgr.y (mgr.14152) 256 : cluster 0 pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:35.526 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-07T03:03:35.526 INFO:teuthology.orchestra.run.smithi189.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-07T03:03:35.526 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10+0 records in 2024-03-07T03:03:35.526 INFO:teuthology.orchestra.run.smithi189.stderr:10+0 records out 2024-03-07T03:03:35.526 INFO:teuthology.orchestra.run.smithi189.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0152198 s, 689 MB/s 2024-03-07T03:03:35.526 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping successful for: 2024-03-07T03:03:35.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:35 smithi145 bash[21339]: cluster 2024-03-07T03:03:34.287692+0000 mon.a (mon.0) 538 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-07T03:03:35.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:35 smithi145 bash[21339]: cluster 2024-03-07T03:03:34.287748+0000 mon.a (mon.0) 539 : cluster 1 Cluster is now healthy 2024-03-07T03:03:35.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:35 smithi145 bash[30906]: cluster 2024-03-07T03:03:34.287692+0000 mon.a (mon.0) 538 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-07T03:03:35.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:35 smithi145 bash[30906]: cluster 2024-03-07T03:03:34.287748+0000 mon.a (mon.0) 539 : cluster 1 Cluster is now healthy 2024-03-07T03:03:35.569 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:35 smithi189 bash[23614]: cluster 2024-03-07T03:03:34.287692+0000 mon.a (mon.0) 538 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-07T03:03:35.569 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:35 smithi189 bash[23614]: cluster 2024-03-07T03:03:34.287748+0000 mon.a (mon.0) 539 : cluster 1 Cluster is now healthy 2024-03-07T03:03:36.248 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi189:vg_nvme/lv_2 2024-03-07T03:03:36.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:36 smithi145 bash[21339]: cluster 2024-03-07T03:03:35.315576+0000 mgr.y (mgr.14152) 257 : cluster 0 pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-03-07T03:03:36.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:36 smithi145 bash[30906]: cluster 2024-03-07T03:03:35.315576+0000 mgr.y (mgr.14152) 257 : cluster 0 pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-03-07T03:03:36.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:36 smithi189 bash[23614]: cluster 2024-03-07T03:03:35.315576+0000 mgr.y (mgr.14152) 257 : cluster 0 pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-03-07T03:03:37.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:37 smithi189 bash[23614]: cephadm 2024-03-07T03:03:36.571777+0000 mgr.y (mgr.14152) 258 : cephadm 1 Detected new or changed devices on smithi189 2024-03-07T03:03:37.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:37 smithi189 bash[23614]: audit 2024-03-07T03:03:36.581179+0000 mon.a (mon.0) 540 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:37.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:37 smithi189 bash[23614]: audit 2024-03-07T03:03:36.592664+0000 mon.a (mon.0) 541 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:37.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:37 smithi189 bash[23614]: audit 2024-03-07T03:03:36.595028+0000 mon.a (mon.0) 542 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:37.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:37 smithi189 bash[23614]: audit 2024-03-07T03:03:36.596862+0000 mon.a (mon.0) 543 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:37.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:37 smithi189 bash[23614]: cephadm 2024-03-07T03:03:36.598223+0000 mgr.y (mgr.14152) 259 : cephadm 1 Adjusting osd_memory_target on smithi189 to 8632M 2024-03-07T03:03:37.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:37 smithi189 bash[23614]: audit 2024-03-07T03:03:36.608692+0000 mon.a (mon.0) 544 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:37.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:37 smithi189 bash[23614]: audit 2024-03-07T03:03:36.610971+0000 mon.a (mon.0) 545 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:37.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:37 smithi189 bash[23614]: audit 2024-03-07T03:03:36.612877+0000 mon.a (mon.0) 546 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:37.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:37 smithi189 bash[23614]: audit 2024-03-07T03:03:36.625058+0000 mon.a (mon.0) 547 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:38.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[21339]: cephadm 2024-03-07T03:03:36.571777+0000 mgr.y (mgr.14152) 258 : cephadm 1 Detected new or changed devices on smithi189 2024-03-07T03:03:38.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[21339]: audit 2024-03-07T03:03:36.581179+0000 mon.a (mon.0) 540 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:38.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[21339]: audit 2024-03-07T03:03:36.592664+0000 mon.a (mon.0) 541 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:38.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[21339]: audit 2024-03-07T03:03:36.595028+0000 mon.a (mon.0) 542 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:38.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[21339]: audit 2024-03-07T03:03:36.596862+0000 mon.a (mon.0) 543 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:38.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[21339]: cephadm 2024-03-07T03:03:36.598223+0000 mgr.y (mgr.14152) 259 : cephadm 1 Adjusting osd_memory_target on smithi189 to 8632M 2024-03-07T03:03:38.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[21339]: audit 2024-03-07T03:03:36.608692+0000 mon.a (mon.0) 544 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:38.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[21339]: audit 2024-03-07T03:03:36.610971+0000 mon.a (mon.0) 545 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:38.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[21339]: audit 2024-03-07T03:03:36.612877+0000 mon.a (mon.0) 546 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:38.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[21339]: audit 2024-03-07T03:03:36.625058+0000 mon.a (mon.0) 547 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:38.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[30906]: cephadm 2024-03-07T03:03:36.571777+0000 mgr.y (mgr.14152) 258 : cephadm 1 Detected new or changed devices on smithi189 2024-03-07T03:03:38.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[30906]: audit 2024-03-07T03:03:36.581179+0000 mon.a (mon.0) 540 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:38.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[30906]: audit 2024-03-07T03:03:36.592664+0000 mon.a (mon.0) 541 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:38.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[30906]: audit 2024-03-07T03:03:36.595028+0000 mon.a (mon.0) 542 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:38.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[30906]: audit 2024-03-07T03:03:36.596862+0000 mon.a (mon.0) 543 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:38.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[30906]: cephadm 2024-03-07T03:03:36.598223+0000 mgr.y (mgr.14152) 259 : cephadm 1 Adjusting osd_memory_target on smithi189 to 8632M 2024-03-07T03:03:38.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[30906]: audit 2024-03-07T03:03:36.608692+0000 mon.a (mon.0) 544 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:38.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[30906]: audit 2024-03-07T03:03:36.610971+0000 mon.a (mon.0) 545 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:38.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[30906]: audit 2024-03-07T03:03:36.612877+0000 mon.a (mon.0) 546 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:38.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:37 smithi145 bash[30906]: audit 2024-03-07T03:03:36.625058+0000 mon.a (mon.0) 547 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:03:39.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:39 smithi145 bash[30906]: cluster 2024-03-07T03:03:37.316132+0000 mgr.y (mgr.14152) 260 : cluster 0 pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-07T03:03:39.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:39 smithi145 bash[21339]: cluster 2024-03-07T03:03:37.316132+0000 mgr.y (mgr.14152) 260 : cluster 0 pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-07T03:03:39.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:39 smithi189 bash[23614]: cluster 2024-03-07T03:03:37.316132+0000 mgr.y (mgr.14152) 260 : cluster 0 pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-07T03:03:40.931 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:03:41.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:41 smithi145 bash[21339]: cluster 2024-03-07T03:03:39.316705+0000 mgr.y (mgr.14152) 261 : cluster 0 pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-03-07T03:03:41.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:41 smithi145 bash[30906]: cluster 2024-03-07T03:03:39.316705+0000 mgr.y (mgr.14152) 261 : cluster 0 pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-03-07T03:03:41.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:41 smithi189 bash[23614]: cluster 2024-03-07T03:03:39.316705+0000 mgr.y (mgr.14152) 261 : cluster 0 pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-03-07T03:03:43.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:43 smithi145 bash[21339]: cluster 2024-03-07T03:03:41.317301+0000 mgr.y (mgr.14152) 262 : cluster 0 pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-03-07T03:03:43.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:43 smithi145 bash[30906]: cluster 2024-03-07T03:03:41.317301+0000 mgr.y (mgr.14152) 262 : cluster 0 pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-03-07T03:03:43.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:43 smithi189 bash[23614]: cluster 2024-03-07T03:03:41.317301+0000 mgr.y (mgr.14152) 262 : cluster 0 pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-03-07T03:03:44.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:44 smithi145 bash[21339]: audit 2024-03-07T03:03:43.365944+0000 mon.a (mon.0) 548 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:03:44.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:44 smithi145 bash[21339]: audit 2024-03-07T03:03:43.372533+0000 mon.a (mon.0) 549 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:03:44.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:44 smithi145 bash[21339]: audit 2024-03-07T03:03:43.374266+0000 mon.a (mon.0) 550 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:44.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:44 smithi145 bash[30906]: audit 2024-03-07T03:03:43.365944+0000 mon.a (mon.0) 548 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:03:44.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:44 smithi145 bash[30906]: audit 2024-03-07T03:03:43.372533+0000 mon.a (mon.0) 549 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:03:44.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:44 smithi145 bash[30906]: audit 2024-03-07T03:03:43.374266+0000 mon.a (mon.0) 550 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:44.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:44 smithi189 bash[23614]: audit 2024-03-07T03:03:43.365944+0000 mon.a (mon.0) 548 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:03:44.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:44 smithi189 bash[23614]: audit 2024-03-07T03:03:43.372533+0000 mon.a (mon.0) 549 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:03:44.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:44 smithi189 bash[23614]: audit 2024-03-07T03:03:43.374266+0000 mon.a (mon.0) 550 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:45.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:45 smithi145 bash[21339]: cluster 2024-03-07T03:03:43.317884+0000 mgr.y (mgr.14152) 263 : cluster 0 pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:03:45.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:45 smithi145 bash[21339]: audit 2024-03-07T03:03:43.358979+0000 mgr.y (mgr.14152) 264 : audit 0 from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:03:45.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:45 smithi145 bash[30906]: cluster 2024-03-07T03:03:43.317884+0000 mgr.y (mgr.14152) 263 : cluster 0 pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:03:45.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:45 smithi145 bash[30906]: audit 2024-03-07T03:03:43.358979+0000 mgr.y (mgr.14152) 264 : audit 0 from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:03:45.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:45 smithi189 bash[23614]: cluster 2024-03-07T03:03:43.317884+0000 mgr.y (mgr.14152) 263 : cluster 0 pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:03:45.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:45 smithi189 bash[23614]: audit 2024-03-07T03:03:43.358979+0000 mgr.y (mgr.14152) 264 : audit 0 from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:03:47.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:47 smithi145 bash[21339]: cluster 2024-03-07T03:03:45.318450+0000 mgr.y (mgr.14152) 265 : cluster 0 pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:47.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:47 smithi145 bash[30906]: cluster 2024-03-07T03:03:45.318450+0000 mgr.y (mgr.14152) 265 : cluster 0 pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:47.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:47 smithi189 bash[23614]: cluster 2024-03-07T03:03:45.318450+0000 mgr.y (mgr.14152) 265 : cluster 0 pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:49.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:49 smithi145 bash[21339]: cluster 2024-03-07T03:03:47.319019+0000 mgr.y (mgr.14152) 266 : cluster 0 pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:49.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:49 smithi145 bash[30906]: cluster 2024-03-07T03:03:47.319019+0000 mgr.y (mgr.14152) 266 : cluster 0 pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:49.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:49 smithi189 bash[23614]: cluster 2024-03-07T03:03:47.319019+0000 mgr.y (mgr.14152) 266 : cluster 0 pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:51.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:51 smithi145 bash[21339]: cluster 2024-03-07T03:03:49.319663+0000 mgr.y (mgr.14152) 267 : cluster 0 pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:51.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:51 smithi145 bash[21339]: audit 2024-03-07T03:03:51.232099+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.189:0/173790522' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f2b456e-712e-45f1-8d84-0fdc4a6d860d"}]: dispatch 2024-03-07T03:03:51.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:51 smithi145 bash[21339]: audit 2024-03-07T03:03:51.232712+0000 mon.a (mon.0) 551 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f2b456e-712e-45f1-8d84-0fdc4a6d860d"}]: dispatch 2024-03-07T03:03:51.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:51 smithi145 bash[21339]: audit 2024-03-07T03:03:51.241689+0000 mon.a (mon.0) 552 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f2b456e-712e-45f1-8d84-0fdc4a6d860d"}]': finished 2024-03-07T03:03:51.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:51 smithi145 bash[21339]: cluster 2024-03-07T03:03:51.241800+0000 mon.a (mon.0) 553 : cluster 0 osdmap e44: 7 total, 6 up, 7 in 2024-03-07T03:03:51.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:51 smithi145 bash[21339]: audit 2024-03-07T03:03:51.242136+0000 mon.a (mon.0) 554 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:03:51.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:51 smithi145 bash[30906]: cluster 2024-03-07T03:03:49.319663+0000 mgr.y (mgr.14152) 267 : cluster 0 pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:51.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:51 smithi145 bash[30906]: audit 2024-03-07T03:03:51.232099+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.189:0/173790522' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f2b456e-712e-45f1-8d84-0fdc4a6d860d"}]: dispatch 2024-03-07T03:03:51.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:51 smithi145 bash[30906]: audit 2024-03-07T03:03:51.232712+0000 mon.a (mon.0) 551 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f2b456e-712e-45f1-8d84-0fdc4a6d860d"}]: dispatch 2024-03-07T03:03:51.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:51 smithi145 bash[30906]: audit 2024-03-07T03:03:51.241689+0000 mon.a (mon.0) 552 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f2b456e-712e-45f1-8d84-0fdc4a6d860d"}]': finished 2024-03-07T03:03:51.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:51 smithi145 bash[30906]: cluster 2024-03-07T03:03:51.241800+0000 mon.a (mon.0) 553 : cluster 0 osdmap e44: 7 total, 6 up, 7 in 2024-03-07T03:03:51.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:51 smithi145 bash[30906]: audit 2024-03-07T03:03:51.242136+0000 mon.a (mon.0) 554 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:03:51.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:51 smithi189 bash[23614]: cluster 2024-03-07T03:03:49.319663+0000 mgr.y (mgr.14152) 267 : cluster 0 pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:51.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:51 smithi189 bash[23614]: audit 2024-03-07T03:03:51.232099+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.189:0/173790522' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f2b456e-712e-45f1-8d84-0fdc4a6d860d"}]: dispatch 2024-03-07T03:03:51.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:51 smithi189 bash[23614]: audit 2024-03-07T03:03:51.232712+0000 mon.a (mon.0) 551 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f2b456e-712e-45f1-8d84-0fdc4a6d860d"}]: dispatch 2024-03-07T03:03:51.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:51 smithi189 bash[23614]: audit 2024-03-07T03:03:51.241689+0000 mon.a (mon.0) 552 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f2b456e-712e-45f1-8d84-0fdc4a6d860d"}]': finished 2024-03-07T03:03:51.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:51 smithi189 bash[23614]: cluster 2024-03-07T03:03:51.241800+0000 mon.a (mon.0) 553 : cluster 0 osdmap e44: 7 total, 6 up, 7 in 2024-03-07T03:03:51.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:51 smithi189 bash[23614]: audit 2024-03-07T03:03:51.242136+0000 mon.a (mon.0) 554 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:03:52.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:52 smithi145 bash[30906]: audit 2024-03-07T03:03:52.015142+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.189:0/3474667920' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:03:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:52 smithi145 bash[21339]: audit 2024-03-07T03:03:52.015142+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.189:0/3474667920' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:03:52.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:52 smithi189 bash[23614]: audit 2024-03-07T03:03:52.015142+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.189:0/3474667920' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:03:53.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:53 smithi145 bash[21339]: cluster 2024-03-07T03:03:51.320211+0000 mgr.y (mgr.14152) 268 : cluster 0 pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:53.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:53 smithi145 bash[30906]: cluster 2024-03-07T03:03:51.320211+0000 mgr.y (mgr.14152) 268 : cluster 0 pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:53.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:53 smithi189 bash[23614]: cluster 2024-03-07T03:03:51.320211+0000 mgr.y (mgr.14152) 268 : cluster 0 pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:54.563 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:54 smithi189 bash[23614]: cluster 2024-03-07T03:03:53.320778+0000 mgr.y (mgr.14152) 269 : cluster 0 pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:54.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:54 smithi145 bash[21339]: cluster 2024-03-07T03:03:53.320778+0000 mgr.y (mgr.14152) 269 : cluster 0 pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:54.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:54 smithi145 bash[30906]: cluster 2024-03-07T03:03:53.320778+0000 mgr.y (mgr.14152) 269 : cluster 0 pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:56.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:56 smithi145 bash[21339]: cluster 2024-03-07T03:03:55.321305+0000 mgr.y (mgr.14152) 270 : cluster 0 pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:56.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:56 smithi145 bash[30906]: cluster 2024-03-07T03:03:55.321305+0000 mgr.y (mgr.14152) 270 : cluster 0 pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:56.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:56 smithi189 bash[23614]: cluster 2024-03-07T03:03:55.321305+0000 mgr.y (mgr.14152) 270 : cluster 0 pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:59.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:03:59 smithi145 bash[21339]: cluster 2024-03-07T03:03:57.321864+0000 mgr.y (mgr.14152) 271 : cluster 0 pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:59.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:03:59 smithi145 bash[30906]: cluster 2024-03-07T03:03:57.321864+0000 mgr.y (mgr.14152) 271 : cluster 0 pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:03:59.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:03:59 smithi189 bash[23614]: cluster 2024-03-07T03:03:57.321864+0000 mgr.y (mgr.14152) 271 : cluster 0 pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:01.310 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:01 smithi189 bash[23614]: cluster 2024-03-07T03:03:59.322523+0000 mgr.y (mgr.14152) 272 : cluster 0 pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:01.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:01 smithi145 bash[21339]: cluster 2024-03-07T03:03:59.322523+0000 mgr.y (mgr.14152) 272 : cluster 0 pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:01.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:01 smithi145 bash[30906]: cluster 2024-03-07T03:03:59.322523+0000 mgr.y (mgr.14152) 272 : cluster 0 pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:03.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:03 smithi145 bash[21339]: cluster 2024-03-07T03:04:01.323216+0000 mgr.y (mgr.14152) 273 : cluster 0 pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:03.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:03 smithi145 bash[21339]: audit 2024-03-07T03:04:03.042874+0000 mon.a (mon.0) 555 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-07T03:04:03.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:03 smithi145 bash[21339]: audit 2024-03-07T03:04:03.044722+0000 mon.a (mon.0) 556 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:03.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:03 smithi145 bash[30906]: cluster 2024-03-07T03:04:01.323216+0000 mgr.y (mgr.14152) 273 : cluster 0 pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:03.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:03 smithi145 bash[30906]: audit 2024-03-07T03:04:03.042874+0000 mon.a (mon.0) 555 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-07T03:04:03.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:03 smithi145 bash[30906]: audit 2024-03-07T03:04:03.044722+0000 mon.a (mon.0) 556 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:03.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:03 smithi189 bash[23614]: cluster 2024-03-07T03:04:01.323216+0000 mgr.y (mgr.14152) 273 : cluster 0 pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:03.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:03 smithi189 bash[23614]: audit 2024-03-07T03:04:03.042874+0000 mon.a (mon.0) 555 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-07T03:04:03.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:03 smithi189 bash[23614]: audit 2024-03-07T03:04:03.044722+0000 mon.a (mon.0) 556 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:04.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:04 smithi189 bash[23614]: cephadm 2024-03-07T03:04:03.046385+0000 mgr.y (mgr.14152) 274 : cephadm 1 Deploying daemon osd.6 on smithi189 2024-03-07T03:04:04.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:04 smithi145 bash[21339]: cephadm 2024-03-07T03:04:03.046385+0000 mgr.y (mgr.14152) 274 : cephadm 1 Deploying daemon osd.6 on smithi189 2024-03-07T03:04:04.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:04 smithi145 bash[30906]: cephadm 2024-03-07T03:04:03.046385+0000 mgr.y (mgr.14152) 274 : cephadm 1 Deploying daemon osd.6 on smithi189 2024-03-07T03:04:05.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:05 smithi145 bash[21339]: cluster 2024-03-07T03:04:03.323787+0000 mgr.y (mgr.14152) 275 : cluster 0 pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:05.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:05 smithi145 bash[30906]: cluster 2024-03-07T03:04:03.323787+0000 mgr.y (mgr.14152) 275 : cluster 0 pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:05.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:05 smithi189 bash[23614]: cluster 2024-03-07T03:04:03.323787+0000 mgr.y (mgr.14152) 275 : cluster 0 pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:06.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:06 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:06.829 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:04:06 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:06.829 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:04:06 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:06.829 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:04:06 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:07.088 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:04:06 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:07.088 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:04:06 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:07.088 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:04:06 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:07.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:06 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:07.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:07 smithi145 bash[21339]: cluster 2024-03-07T03:04:05.324381+0000 mgr.y (mgr.14152) 276 : cluster 0 pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:07.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:07 smithi145 bash[21339]: audit 2024-03-07T03:04:07.095717+0000 mon.a (mon.0) 557 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:07.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:07 smithi145 bash[21339]: audit 2024-03-07T03:04:07.096184+0000 mon.a (mon.0) 558 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:07.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:07 smithi145 bash[21339]: audit 2024-03-07T03:04:07.105908+0000 mon.a (mon.0) 559 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:07.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:07 smithi145 bash[30906]: cluster 2024-03-07T03:04:05.324381+0000 mgr.y (mgr.14152) 276 : cluster 0 pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:07.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:07 smithi145 bash[30906]: audit 2024-03-07T03:04:07.095717+0000 mon.a (mon.0) 557 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:07.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:07 smithi145 bash[30906]: audit 2024-03-07T03:04:07.096184+0000 mon.a (mon.0) 558 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:07.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:07 smithi145 bash[30906]: audit 2024-03-07T03:04:07.105908+0000 mon.a (mon.0) 559 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:07.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:07 smithi189 bash[23614]: cluster 2024-03-07T03:04:05.324381+0000 mgr.y (mgr.14152) 276 : cluster 0 pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:07.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:07 smithi189 bash[23614]: audit 2024-03-07T03:04:07.095717+0000 mon.a (mon.0) 557 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:07.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:07 smithi189 bash[23614]: audit 2024-03-07T03:04:07.096184+0000 mon.a (mon.0) 558 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:07.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:07 smithi189 bash[23614]: audit 2024-03-07T03:04:07.105908+0000 mon.a (mon.0) 559 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:09.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:09 smithi189 bash[23614]: cluster 2024-03-07T03:04:07.324917+0000 mgr.y (mgr.14152) 277 : cluster 0 pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:09.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:09 smithi145 bash[21339]: cluster 2024-03-07T03:04:07.324917+0000 mgr.y (mgr.14152) 277 : cluster 0 pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:09.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:09 smithi145 bash[30906]: cluster 2024-03-07T03:04:07.324917+0000 mgr.y (mgr.14152) 277 : cluster 0 pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:11.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:11 smithi145 bash[21339]: cluster 2024-03-07T03:04:09.325445+0000 mgr.y (mgr.14152) 278 : cluster 0 pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:11.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:11 smithi145 bash[30906]: cluster 2024-03-07T03:04:09.325445+0000 mgr.y (mgr.14152) 278 : cluster 0 pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:11.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:11 smithi189 bash[23614]: cluster 2024-03-07T03:04:09.325445+0000 mgr.y (mgr.14152) 278 : cluster 0 pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:12.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:12 smithi145 bash[21339]: cluster 2024-03-07T03:04:11.326021+0000 mgr.y (mgr.14152) 279 : cluster 0 pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:12.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:12 smithi145 bash[30906]: cluster 2024-03-07T03:04:11.326021+0000 mgr.y (mgr.14152) 279 : cluster 0 pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:12.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:12 smithi189 bash[23614]: cluster 2024-03-07T03:04:11.326021+0000 mgr.y (mgr.14152) 279 : cluster 0 pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:13.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:13 smithi189 bash[23614]: audit 2024-03-07T03:04:12.359601+0000 mon.a (mon.0) 560 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:13.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:13 smithi189 bash[23614]: audit 2024-03-07T03:04:12.369485+0000 mon.a (mon.0) 561 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:13.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:13 smithi189 bash[23614]: audit 2024-03-07T03:04:12.895268+0000 mon.a (mon.0) 562 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:13.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:13 smithi189 bash[23614]: audit 2024-03-07T03:04:12.897148+0000 mon.a (mon.0) 563 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:04:13.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:13 smithi189 bash[23614]: audit 2024-03-07T03:04:12.907441+0000 mon.a (mon.0) 564 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:13.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:13 smithi189 bash[23614]: audit 2024-03-07T03:04:13.321352+0000 mon.b (mon.1) 20 : audit 1 from='osd.6 [v2:172.21.15.189:6816/4219543665,v1:172.21.15.189:6817/4219543665]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-07T03:04:13.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:13 smithi189 bash[23614]: audit 2024-03-07T03:04:13.322386+0000 mon.a (mon.0) 565 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-07T03:04:13.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[21339]: audit 2024-03-07T03:04:12.359601+0000 mon.a (mon.0) 560 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:13.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[21339]: audit 2024-03-07T03:04:12.369485+0000 mon.a (mon.0) 561 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:13.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[21339]: audit 2024-03-07T03:04:12.895268+0000 mon.a (mon.0) 562 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:13.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[21339]: audit 2024-03-07T03:04:12.897148+0000 mon.a (mon.0) 563 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:04:13.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[21339]: audit 2024-03-07T03:04:12.907441+0000 mon.a (mon.0) 564 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:13.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[21339]: audit 2024-03-07T03:04:13.321352+0000 mon.b (mon.1) 20 : audit 1 from='osd.6 [v2:172.21.15.189:6816/4219543665,v1:172.21.15.189:6817/4219543665]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-07T03:04:13.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[21339]: audit 2024-03-07T03:04:13.322386+0000 mon.a (mon.0) 565 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-07T03:04:13.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[30906]: audit 2024-03-07T03:04:12.359601+0000 mon.a (mon.0) 560 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:13.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[30906]: audit 2024-03-07T03:04:12.369485+0000 mon.a (mon.0) 561 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:13.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[30906]: audit 2024-03-07T03:04:12.895268+0000 mon.a (mon.0) 562 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:13.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[30906]: audit 2024-03-07T03:04:12.897148+0000 mon.a (mon.0) 563 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:04:13.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[30906]: audit 2024-03-07T03:04:12.907441+0000 mon.a (mon.0) 564 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:13.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[30906]: audit 2024-03-07T03:04:13.321352+0000 mon.b (mon.1) 20 : audit 1 from='osd.6 [v2:172.21.15.189:6816/4219543665,v1:172.21.15.189:6817/4219543665]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-07T03:04:13.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:13 smithi145 bash[30906]: audit 2024-03-07T03:04:13.322386+0000 mon.a (mon.0) 565 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-07T03:04:15.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:15 smithi145 bash[21339]: cluster 2024-03-07T03:04:13.326653+0000 mgr.y (mgr.14152) 280 : cluster 0 pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:15.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:15 smithi145 bash[21339]: audit 2024-03-07T03:04:14.279525+0000 mon.a (mon.0) 566 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-07T03:04:15.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:15 smithi145 bash[21339]: cluster 2024-03-07T03:04:14.279666+0000 mon.a (mon.0) 567 : cluster 0 osdmap e45: 7 total, 6 up, 7 in 2024-03-07T03:04:15.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:15 smithi145 bash[21339]: audit 2024-03-07T03:04:14.279918+0000 mon.a (mon.0) 568 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:04:15.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:15 smithi145 bash[21339]: audit 2024-03-07T03:04:14.280668+0000 mon.b (mon.1) 21 : audit 1 from='osd.6 [v2:172.21.15.189:6816/4219543665,v1:172.21.15.189:6817/4219543665]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:04:15.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:15 smithi145 bash[21339]: audit 2024-03-07T03:04:14.281462+0000 mon.a (mon.0) 569 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:04:15.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:15 smithi145 bash[30906]: cluster 2024-03-07T03:04:13.326653+0000 mgr.y (mgr.14152) 280 : cluster 0 pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:15.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:15 smithi145 bash[30906]: audit 2024-03-07T03:04:14.279525+0000 mon.a (mon.0) 566 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-07T03:04:15.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:15 smithi145 bash[30906]: cluster 2024-03-07T03:04:14.279666+0000 mon.a (mon.0) 567 : cluster 0 osdmap e45: 7 total, 6 up, 7 in 2024-03-07T03:04:15.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:15 smithi145 bash[30906]: audit 2024-03-07T03:04:14.279918+0000 mon.a (mon.0) 568 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:04:15.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:15 smithi145 bash[30906]: audit 2024-03-07T03:04:14.280668+0000 mon.b (mon.1) 21 : audit 1 from='osd.6 [v2:172.21.15.189:6816/4219543665,v1:172.21.15.189:6817/4219543665]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:04:15.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:15 smithi145 bash[30906]: audit 2024-03-07T03:04:14.281462+0000 mon.a (mon.0) 569 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:04:15.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:15 smithi189 bash[23614]: cluster 2024-03-07T03:04:13.326653+0000 mgr.y (mgr.14152) 280 : cluster 0 pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:15.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:15 smithi189 bash[23614]: audit 2024-03-07T03:04:14.279525+0000 mon.a (mon.0) 566 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-07T03:04:15.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:15 smithi189 bash[23614]: cluster 2024-03-07T03:04:14.279666+0000 mon.a (mon.0) 567 : cluster 0 osdmap e45: 7 total, 6 up, 7 in 2024-03-07T03:04:15.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:15 smithi189 bash[23614]: audit 2024-03-07T03:04:14.279918+0000 mon.a (mon.0) 568 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:04:15.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:15 smithi189 bash[23614]: audit 2024-03-07T03:04:14.280668+0000 mon.b (mon.1) 21 : audit 1 from='osd.6 [v2:172.21.15.189:6816/4219543665,v1:172.21.15.189:6817/4219543665]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:04:15.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:15 smithi189 bash[23614]: audit 2024-03-07T03:04:14.281462+0000 mon.a (mon.0) 569 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:04:15.592 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 6 on host 'smithi189' 2024-03-07T03:04:16.218 DEBUG:teuthology.orchestra.run.smithi189:osd.6> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.6.service 2024-03-07T03:04:16.221 INFO:tasks.cephadm:Deploying osd.7 on smithi189 with /dev/vg_nvme/lv_1... 2024-03-07T03:04:16.221 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-07T03:04:16.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[21339]: audit 2024-03-07T03:04:15.282608+0000 mon.a (mon.0) 570 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-07T03:04:16.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[21339]: cluster 2024-03-07T03:04:15.282940+0000 mon.a (mon.0) 571 : cluster 0 osdmap e46: 7 total, 6 up, 7 in 2024-03-07T03:04:16.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[21339]: audit 2024-03-07T03:04:15.283694+0000 mon.a (mon.0) 572 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:04:16.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[21339]: audit 2024-03-07T03:04:15.293228+0000 mon.a (mon.0) 573 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:04:16.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[21339]: audit 2024-03-07T03:04:15.544374+0000 mon.a (mon.0) 574 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:16.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[21339]: audit 2024-03-07T03:04:15.548760+0000 mon.a (mon.0) 575 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:16.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[21339]: audit 2024-03-07T03:04:15.554875+0000 mon.a (mon.0) 576 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[30906]: audit 2024-03-07T03:04:15.282608+0000 mon.a (mon.0) 570 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-07T03:04:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[30906]: cluster 2024-03-07T03:04:15.282940+0000 mon.a (mon.0) 571 : cluster 0 osdmap e46: 7 total, 6 up, 7 in 2024-03-07T03:04:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[30906]: audit 2024-03-07T03:04:15.283694+0000 mon.a (mon.0) 572 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:04:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[30906]: audit 2024-03-07T03:04:15.293228+0000 mon.a (mon.0) 573 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:04:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[30906]: audit 2024-03-07T03:04:15.544374+0000 mon.a (mon.0) 574 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[30906]: audit 2024-03-07T03:04:15.548760+0000 mon.a (mon.0) 575 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:16 smithi145 bash[30906]: audit 2024-03-07T03:04:15.554875+0000 mon.a (mon.0) 576 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:16.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:16 smithi189 bash[23614]: audit 2024-03-07T03:04:15.282608+0000 mon.a (mon.0) 570 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-07T03:04:16.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:16 smithi189 bash[23614]: cluster 2024-03-07T03:04:15.282940+0000 mon.a (mon.0) 571 : cluster 0 osdmap e46: 7 total, 6 up, 7 in 2024-03-07T03:04:16.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:16 smithi189 bash[23614]: audit 2024-03-07T03:04:15.283694+0000 mon.a (mon.0) 572 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:04:16.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:16 smithi189 bash[23614]: audit 2024-03-07T03:04:15.293228+0000 mon.a (mon.0) 573 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:04:16.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:16 smithi189 bash[23614]: audit 2024-03-07T03:04:15.544374+0000 mon.a (mon.0) 574 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:16.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:16 smithi189 bash[23614]: audit 2024-03-07T03:04:15.548760+0000 mon.a (mon.0) 575 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:16.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:16 smithi189 bash[23614]: audit 2024-03-07T03:04:15.554875+0000 mon.a (mon.0) 576 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:17.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:17 smithi145 bash[30906]: cluster 2024-03-07T03:04:14.345018+0000 osd.6 (osd.6) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:04:17.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:17 smithi145 bash[30906]: cluster 2024-03-07T03:04:14.345167+0000 osd.6 (osd.6) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:04:17.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:17 smithi145 bash[30906]: cluster 2024-03-07T03:04:15.327226+0000 mgr.y (mgr.14152) 281 : cluster 0 pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:17.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:17 smithi145 bash[30906]: cluster 2024-03-07T03:04:16.294710+0000 mon.a (mon.0) 577 : cluster 1 osd.6 [v2:172.21.15.189:6816/4219543665,v1:172.21.15.189:6817/4219543665] boot 2024-03-07T03:04:17.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:17 smithi145 bash[30906]: cluster 2024-03-07T03:04:16.294919+0000 mon.a (mon.0) 578 : cluster 0 osdmap e47: 7 total, 7 up, 7 in 2024-03-07T03:04:17.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:17 smithi145 bash[30906]: audit 2024-03-07T03:04:16.295679+0000 mon.a (mon.0) 579 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:04:17.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:17 smithi145 bash[21339]: cluster 2024-03-07T03:04:14.345018+0000 osd.6 (osd.6) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:04:17.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:17 smithi145 bash[21339]: cluster 2024-03-07T03:04:14.345167+0000 osd.6 (osd.6) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:04:17.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:17 smithi145 bash[21339]: cluster 2024-03-07T03:04:15.327226+0000 mgr.y (mgr.14152) 281 : cluster 0 pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:17.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:17 smithi145 bash[21339]: cluster 2024-03-07T03:04:16.294710+0000 mon.a (mon.0) 577 : cluster 1 osd.6 [v2:172.21.15.189:6816/4219543665,v1:172.21.15.189:6817/4219543665] boot 2024-03-07T03:04:17.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:17 smithi145 bash[21339]: cluster 2024-03-07T03:04:16.294919+0000 mon.a (mon.0) 578 : cluster 0 osdmap e47: 7 total, 7 up, 7 in 2024-03-07T03:04:17.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:17 smithi145 bash[21339]: audit 2024-03-07T03:04:16.295679+0000 mon.a (mon.0) 579 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:04:17.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:17 smithi189 bash[23614]: cluster 2024-03-07T03:04:14.345018+0000 osd.6 (osd.6) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:04:17.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:17 smithi189 bash[23614]: cluster 2024-03-07T03:04:14.345167+0000 osd.6 (osd.6) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:04:17.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:17 smithi189 bash[23614]: cluster 2024-03-07T03:04:15.327226+0000 mgr.y (mgr.14152) 281 : cluster 0 pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:04:17.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:17 smithi189 bash[23614]: cluster 2024-03-07T03:04:16.294710+0000 mon.a (mon.0) 577 : cluster 1 osd.6 [v2:172.21.15.189:6816/4219543665,v1:172.21.15.189:6817/4219543665] boot 2024-03-07T03:04:17.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:17 smithi189 bash[23614]: cluster 2024-03-07T03:04:16.294919+0000 mon.a (mon.0) 578 : cluster 0 osdmap e47: 7 total, 7 up, 7 in 2024-03-07T03:04:17.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:17 smithi189 bash[23614]: audit 2024-03-07T03:04:16.295679+0000 mon.a (mon.0) 579 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:04:18.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:18 smithi145 bash[21339]: cluster 2024-03-07T03:04:17.306621+0000 mon.a (mon.0) 580 : cluster 0 osdmap e48: 7 total, 7 up, 7 in 2024-03-07T03:04:18.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:18 smithi145 bash[21339]: cluster 2024-03-07T03:04:17.327754+0000 mgr.y (mgr.14152) 282 : cluster 0 pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:18.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:18 smithi145 bash[21339]: cluster 2024-03-07T03:04:18.308755+0000 mon.a (mon.0) 581 : cluster 0 osdmap e49: 7 total, 7 up, 7 in 2024-03-07T03:04:18.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:18 smithi145 bash[30906]: cluster 2024-03-07T03:04:17.306621+0000 mon.a (mon.0) 580 : cluster 0 osdmap e48: 7 total, 7 up, 7 in 2024-03-07T03:04:18.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:18 smithi145 bash[30906]: cluster 2024-03-07T03:04:17.327754+0000 mgr.y (mgr.14152) 282 : cluster 0 pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:18.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:18 smithi145 bash[30906]: cluster 2024-03-07T03:04:18.308755+0000 mon.a (mon.0) 581 : cluster 0 osdmap e49: 7 total, 7 up, 7 in 2024-03-07T03:04:18.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:18 smithi189 bash[23614]: cluster 2024-03-07T03:04:17.306621+0000 mon.a (mon.0) 580 : cluster 0 osdmap e48: 7 total, 7 up, 7 in 2024-03-07T03:04:18.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:18 smithi189 bash[23614]: cluster 2024-03-07T03:04:17.327754+0000 mgr.y (mgr.14152) 282 : cluster 0 pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:18.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:18 smithi189 bash[23614]: cluster 2024-03-07T03:04:18.308755+0000 mon.a (mon.0) 581 : cluster 0 osdmap e49: 7 total, 7 up, 7 in 2024-03-07T03:04:20.398 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:04:20.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:20 smithi189 bash[23614]: cluster 2024-03-07T03:04:19.327239+0000 mon.a (mon.0) 582 : cluster 0 osdmap e50: 7 total, 7 up, 7 in 2024-03-07T03:04:20.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:20 smithi189 bash[23614]: cluster 2024-03-07T03:04:19.328410+0000 mgr.y (mgr.14152) 283 : cluster 0 pgmap v266: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:20.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:20 smithi145 bash[21339]: cluster 2024-03-07T03:04:19.327239+0000 mon.a (mon.0) 582 : cluster 0 osdmap e50: 7 total, 7 up, 7 in 2024-03-07T03:04:20.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:20 smithi145 bash[21339]: cluster 2024-03-07T03:04:19.328410+0000 mgr.y (mgr.14152) 283 : cluster 0 pgmap v266: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:20.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:20 smithi145 bash[30906]: cluster 2024-03-07T03:04:19.327239+0000 mon.a (mon.0) 582 : cluster 0 osdmap e50: 7 total, 7 up, 7 in 2024-03-07T03:04:20.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:20 smithi145 bash[30906]: cluster 2024-03-07T03:04:19.328410+0000 mgr.y (mgr.14152) 283 : cluster 0 pgmap v266: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:21.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:21 smithi189 bash[23614]: cluster 2024-03-07T03:04:20.338250+0000 mon.a (mon.0) 583 : cluster 0 osdmap e51: 7 total, 7 up, 7 in 2024-03-07T03:04:21.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:21 smithi145 bash[21339]: cluster 2024-03-07T03:04:20.338250+0000 mon.a (mon.0) 583 : cluster 0 osdmap e51: 7 total, 7 up, 7 in 2024-03-07T03:04:21.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:21 smithi145 bash[30906]: cluster 2024-03-07T03:04:20.338250+0000 mon.a (mon.0) 583 : cluster 0 osdmap e51: 7 total, 7 up, 7 in 2024-03-07T03:04:22.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:22 smithi145 bash[21339]: cluster 2024-03-07T03:04:21.328997+0000 mgr.y (mgr.14152) 284 : cluster 0 pgmap v268: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:22.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:22 smithi145 bash[30906]: cluster 2024-03-07T03:04:21.328997+0000 mgr.y (mgr.14152) 284 : cluster 0 pgmap v268: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:22.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:22 smithi189 bash[23614]: cluster 2024-03-07T03:04:21.328997+0000 mgr.y (mgr.14152) 284 : cluster 0 pgmap v268: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:23.919 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-07T03:04:23.920 INFO:teuthology.orchestra.run.smithi189.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-07T03:04:23.920 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10+0 records in 2024-03-07T03:04:23.920 INFO:teuthology.orchestra.run.smithi189.stderr:10+0 records out 2024-03-07T03:04:23.920 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0130574 s, 803 MB/s 2024-03-07T03:04:23.920 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping successful for: 2024-03-07T03:04:24.747 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi189:vg_nvme/lv_1 2024-03-07T03:04:25.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[21339]: cluster 2024-03-07T03:04:23.329617+0000 mgr.y (mgr.14152) 285 : cluster 0 pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-07T03:04:25.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[21339]: audit 2024-03-07T03:04:25.160309+0000 mon.a (mon.0) 584 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:25.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[21339]: audit 2024-03-07T03:04:25.171985+0000 mon.a (mon.0) 585 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:25.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[21339]: audit 2024-03-07T03:04:25.174492+0000 mon.a (mon.0) 586 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:25.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[21339]: audit 2024-03-07T03:04:25.176215+0000 mon.a (mon.0) 587 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:25.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[21339]: audit 2024-03-07T03:04:25.177751+0000 mon.a (mon.0) 588 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:25.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[21339]: audit 2024-03-07T03:04:25.190941+0000 mon.a (mon.0) 589 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:25.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[21339]: audit 2024-03-07T03:04:25.193064+0000 mon.a (mon.0) 590 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:25.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[21339]: audit 2024-03-07T03:04:25.194876+0000 mon.a (mon.0) 591 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:04:25.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[21339]: audit 2024-03-07T03:04:25.205236+0000 mon.a (mon.0) 592 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:25.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[30906]: cluster 2024-03-07T03:04:23.329617+0000 mgr.y (mgr.14152) 285 : cluster 0 pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-07T03:04:25.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[30906]: audit 2024-03-07T03:04:25.160309+0000 mon.a (mon.0) 584 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:25.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[30906]: audit 2024-03-07T03:04:25.171985+0000 mon.a (mon.0) 585 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:25.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[30906]: audit 2024-03-07T03:04:25.174492+0000 mon.a (mon.0) 586 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:25.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[30906]: audit 2024-03-07T03:04:25.176215+0000 mon.a (mon.0) 587 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:25.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[30906]: audit 2024-03-07T03:04:25.177751+0000 mon.a (mon.0) 588 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:25.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[30906]: audit 2024-03-07T03:04:25.190941+0000 mon.a (mon.0) 589 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:25.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[30906]: audit 2024-03-07T03:04:25.193064+0000 mon.a (mon.0) 590 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:25.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[30906]: audit 2024-03-07T03:04:25.194876+0000 mon.a (mon.0) 591 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:04:25.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:25 smithi145 bash[30906]: audit 2024-03-07T03:04:25.205236+0000 mon.a (mon.0) 592 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:25.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:25 smithi189 bash[23614]: cluster 2024-03-07T03:04:23.329617+0000 mgr.y (mgr.14152) 285 : cluster 0 pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-07T03:04:25.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:25 smithi189 bash[23614]: audit 2024-03-07T03:04:25.160309+0000 mon.a (mon.0) 584 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:25.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:25 smithi189 bash[23614]: audit 2024-03-07T03:04:25.171985+0000 mon.a (mon.0) 585 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:25.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:25 smithi189 bash[23614]: audit 2024-03-07T03:04:25.174492+0000 mon.a (mon.0) 586 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:25.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:25 smithi189 bash[23614]: audit 2024-03-07T03:04:25.176215+0000 mon.a (mon.0) 587 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:25.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:25 smithi189 bash[23614]: audit 2024-03-07T03:04:25.177751+0000 mon.a (mon.0) 588 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:25.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:25 smithi189 bash[23614]: audit 2024-03-07T03:04:25.190941+0000 mon.a (mon.0) 589 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:25.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:25 smithi189 bash[23614]: audit 2024-03-07T03:04:25.193064+0000 mon.a (mon.0) 590 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:25.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:25 smithi189 bash[23614]: audit 2024-03-07T03:04:25.194876+0000 mon.a (mon.0) 591 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:04:25.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:25 smithi189 bash[23614]: audit 2024-03-07T03:04:25.205236+0000 mon.a (mon.0) 592 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:26.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:26 smithi145 bash[21339]: cephadm 2024-03-07T03:04:25.148719+0000 mgr.y (mgr.14152) 286 : cephadm 1 Detected new or changed devices on smithi189 2024-03-07T03:04:26.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:26 smithi145 bash[21339]: cephadm 2024-03-07T03:04:25.179160+0000 mgr.y (mgr.14152) 287 : cephadm 1 Adjusting osd_memory_target on smithi189 to 5755M 2024-03-07T03:04:26.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:26 smithi145 bash[30906]: cephadm 2024-03-07T03:04:25.148719+0000 mgr.y (mgr.14152) 286 : cephadm 1 Detected new or changed devices on smithi189 2024-03-07T03:04:26.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:26 smithi145 bash[30906]: cephadm 2024-03-07T03:04:25.179160+0000 mgr.y (mgr.14152) 287 : cephadm 1 Adjusting osd_memory_target on smithi189 to 5755M 2024-03-07T03:04:26.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:26 smithi189 bash[23614]: cephadm 2024-03-07T03:04:25.148719+0000 mgr.y (mgr.14152) 286 : cephadm 1 Detected new or changed devices on smithi189 2024-03-07T03:04:26.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:26 smithi189 bash[23614]: cephadm 2024-03-07T03:04:25.179160+0000 mgr.y (mgr.14152) 287 : cephadm 1 Adjusting osd_memory_target on smithi189 to 5755M 2024-03-07T03:04:27.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:27 smithi145 bash[21339]: cluster 2024-03-07T03:04:25.330255+0000 mgr.y (mgr.14152) 288 : cluster 0 pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-03-07T03:04:27.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:27 smithi145 bash[30906]: cluster 2024-03-07T03:04:25.330255+0000 mgr.y (mgr.14152) 288 : cluster 0 pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-03-07T03:04:27.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:27 smithi189 bash[23614]: cluster 2024-03-07T03:04:25.330255+0000 mgr.y (mgr.14152) 288 : cluster 0 pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-03-07T03:04:29.440 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:04:29.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:29 smithi145 bash[21339]: cluster 2024-03-07T03:04:27.330852+0000 mgr.y (mgr.14152) 289 : cluster 0 pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-07T03:04:29.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:29 smithi145 bash[30906]: cluster 2024-03-07T03:04:27.330852+0000 mgr.y (mgr.14152) 289 : cluster 0 pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-07T03:04:29.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:29 smithi189 bash[23614]: cluster 2024-03-07T03:04:27.330852+0000 mgr.y (mgr.14152) 289 : cluster 0 pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-07T03:04:31.557 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:31 smithi189 bash[23614]: cluster 2024-03-07T03:04:29.331470+0000 mgr.y (mgr.14152) 290 : cluster 0 pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:04:31.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:31 smithi145 bash[21339]: cluster 2024-03-07T03:04:29.331470+0000 mgr.y (mgr.14152) 290 : cluster 0 pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:04:31.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:31 smithi145 bash[30906]: cluster 2024-03-07T03:04:29.331470+0000 mgr.y (mgr.14152) 290 : cluster 0 pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:04:32.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:32 smithi145 bash[21339]: audit 2024-03-07T03:04:31.979925+0000 mon.a (mon.0) 593 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:04:32.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:32 smithi145 bash[21339]: audit 2024-03-07T03:04:31.986297+0000 mon.a (mon.0) 594 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:04:32.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:32 smithi145 bash[21339]: audit 2024-03-07T03:04:31.987926+0000 mon.a (mon.0) 595 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:32.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:32 smithi145 bash[30906]: audit 2024-03-07T03:04:31.979925+0000 mon.a (mon.0) 593 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:04:32.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:32 smithi145 bash[30906]: audit 2024-03-07T03:04:31.986297+0000 mon.a (mon.0) 594 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:04:32.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:32 smithi145 bash[30906]: audit 2024-03-07T03:04:31.987926+0000 mon.a (mon.0) 595 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:32 smithi189 bash[23614]: audit 2024-03-07T03:04:31.979925+0000 mon.a (mon.0) 593 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:04:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:32 smithi189 bash[23614]: audit 2024-03-07T03:04:31.986297+0000 mon.a (mon.0) 594 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:04:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:32 smithi189 bash[23614]: audit 2024-03-07T03:04:31.987926+0000 mon.a (mon.0) 595 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:33.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:33 smithi145 bash[30906]: cluster 2024-03-07T03:04:31.332026+0000 mgr.y (mgr.14152) 291 : cluster 0 pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-07T03:04:33.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:33 smithi145 bash[30906]: audit 2024-03-07T03:04:31.973099+0000 mgr.y (mgr.14152) 292 : audit 0 from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:04:33.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:33 smithi145 bash[21339]: cluster 2024-03-07T03:04:31.332026+0000 mgr.y (mgr.14152) 291 : cluster 0 pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-07T03:04:33.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:33 smithi145 bash[21339]: audit 2024-03-07T03:04:31.973099+0000 mgr.y (mgr.14152) 292 : audit 0 from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:04:33.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:33 smithi189 bash[23614]: cluster 2024-03-07T03:04:31.332026+0000 mgr.y (mgr.14152) 291 : cluster 0 pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-07T03:04:33.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:33 smithi189 bash[23614]: audit 2024-03-07T03:04:31.973099+0000 mgr.y (mgr.14152) 292 : audit 0 from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:04:34.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:34 smithi145 bash[30906]: cluster 2024-03-07T03:04:33.332628+0000 mgr.y (mgr.14152) 293 : cluster 0 pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:04:34.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:34 smithi145 bash[21339]: cluster 2024-03-07T03:04:33.332628+0000 mgr.y (mgr.14152) 293 : cluster 0 pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:04:34.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:34 smithi189 bash[23614]: cluster 2024-03-07T03:04:33.332628+0000 mgr.y (mgr.14152) 293 : cluster 0 pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:04:36.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:36 smithi145 bash[30906]: cluster 2024-03-07T03:04:35.333176+0000 mgr.y (mgr.14152) 294 : cluster 0 pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:36.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:36 smithi145 bash[21339]: cluster 2024-03-07T03:04:35.333176+0000 mgr.y (mgr.14152) 294 : cluster 0 pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:36.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:36 smithi189 bash[23614]: cluster 2024-03-07T03:04:35.333176+0000 mgr.y (mgr.14152) 294 : cluster 0 pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:39.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:39 smithi145 bash[21339]: cluster 2024-03-07T03:04:37.333743+0000 mgr.y (mgr.14152) 295 : cluster 0 pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:39.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:39 smithi145 bash[30906]: cluster 2024-03-07T03:04:37.333743+0000 mgr.y (mgr.14152) 295 : cluster 0 pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:39.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:39 smithi189 bash[23614]: cluster 2024-03-07T03:04:37.333743+0000 mgr.y (mgr.14152) 295 : cluster 0 pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:40.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:40 smithi145 bash[21339]: audit 2024-03-07T03:04:39.930210+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.189:0/3134138525' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "24dc7eba-4cd4-414d-93d2-598f5f2e655c"}]: dispatch 2024-03-07T03:04:40.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:40 smithi145 bash[21339]: audit 2024-03-07T03:04:39.931526+0000 mon.a (mon.0) 596 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "24dc7eba-4cd4-414d-93d2-598f5f2e655c"}]: dispatch 2024-03-07T03:04:40.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:40 smithi145 bash[21339]: audit 2024-03-07T03:04:39.940222+0000 mon.a (mon.0) 597 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "24dc7eba-4cd4-414d-93d2-598f5f2e655c"}]': finished 2024-03-07T03:04:40.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:40 smithi145 bash[21339]: cluster 2024-03-07T03:04:39.940358+0000 mon.a (mon.0) 598 : cluster 0 osdmap e52: 8 total, 7 up, 8 in 2024-03-07T03:04:40.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:40 smithi145 bash[21339]: audit 2024-03-07T03:04:39.940622+0000 mon.a (mon.0) 599 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:04:40.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:40 smithi145 bash[30906]: audit 2024-03-07T03:04:39.930210+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.189:0/3134138525' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "24dc7eba-4cd4-414d-93d2-598f5f2e655c"}]: dispatch 2024-03-07T03:04:40.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:40 smithi145 bash[30906]: audit 2024-03-07T03:04:39.931526+0000 mon.a (mon.0) 596 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "24dc7eba-4cd4-414d-93d2-598f5f2e655c"}]: dispatch 2024-03-07T03:04:40.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:40 smithi145 bash[30906]: audit 2024-03-07T03:04:39.940222+0000 mon.a (mon.0) 597 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "24dc7eba-4cd4-414d-93d2-598f5f2e655c"}]': finished 2024-03-07T03:04:40.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:40 smithi145 bash[30906]: cluster 2024-03-07T03:04:39.940358+0000 mon.a (mon.0) 598 : cluster 0 osdmap e52: 8 total, 7 up, 8 in 2024-03-07T03:04:40.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:40 smithi145 bash[30906]: audit 2024-03-07T03:04:39.940622+0000 mon.a (mon.0) 599 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:04:40.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:40 smithi189 bash[23614]: audit 2024-03-07T03:04:39.930210+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.189:0/3134138525' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "24dc7eba-4cd4-414d-93d2-598f5f2e655c"}]: dispatch 2024-03-07T03:04:40.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:40 smithi189 bash[23614]: audit 2024-03-07T03:04:39.931526+0000 mon.a (mon.0) 596 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "24dc7eba-4cd4-414d-93d2-598f5f2e655c"}]: dispatch 2024-03-07T03:04:40.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:40 smithi189 bash[23614]: audit 2024-03-07T03:04:39.940222+0000 mon.a (mon.0) 597 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "24dc7eba-4cd4-414d-93d2-598f5f2e655c"}]': finished 2024-03-07T03:04:40.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:40 smithi189 bash[23614]: cluster 2024-03-07T03:04:39.940358+0000 mon.a (mon.0) 598 : cluster 0 osdmap e52: 8 total, 7 up, 8 in 2024-03-07T03:04:40.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:40 smithi189 bash[23614]: audit 2024-03-07T03:04:39.940622+0000 mon.a (mon.0) 599 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:04:41.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:41 smithi145 bash[21339]: cluster 2024-03-07T03:04:39.334329+0000 mgr.y (mgr.14152) 296 : cluster 0 pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:41.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:41 smithi145 bash[21339]: audit 2024-03-07T03:04:40.670849+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.189:0/458171321' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:04:41.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:41 smithi145 bash[30906]: cluster 2024-03-07T03:04:39.334329+0000 mgr.y (mgr.14152) 296 : cluster 0 pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:41.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:41 smithi145 bash[30906]: audit 2024-03-07T03:04:40.670849+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.189:0/458171321' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:04:41.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:41 smithi189 bash[23614]: cluster 2024-03-07T03:04:39.334329+0000 mgr.y (mgr.14152) 296 : cluster 0 pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:41.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:41 smithi189 bash[23614]: audit 2024-03-07T03:04:40.670849+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.189:0/458171321' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:04:43.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:43 smithi145 bash[21339]: cluster 2024-03-07T03:04:41.334899+0000 mgr.y (mgr.14152) 297 : cluster 0 pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:43.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:43 smithi145 bash[30906]: cluster 2024-03-07T03:04:41.334899+0000 mgr.y (mgr.14152) 297 : cluster 0 pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:43.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:43 smithi189 bash[23614]: cluster 2024-03-07T03:04:41.334899+0000 mgr.y (mgr.14152) 297 : cluster 0 pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:45.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:45 smithi145 bash[21339]: cluster 2024-03-07T03:04:43.335487+0000 mgr.y (mgr.14152) 298 : cluster 0 pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:45.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:45 smithi145 bash[30906]: cluster 2024-03-07T03:04:43.335487+0000 mgr.y (mgr.14152) 298 : cluster 0 pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:45.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:45 smithi189 bash[23614]: cluster 2024-03-07T03:04:43.335487+0000 mgr.y (mgr.14152) 298 : cluster 0 pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:47.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:47 smithi145 bash[21339]: cluster 2024-03-07T03:04:45.336036+0000 mgr.y (mgr.14152) 299 : cluster 0 pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:47.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:47 smithi145 bash[30906]: cluster 2024-03-07T03:04:45.336036+0000 mgr.y (mgr.14152) 299 : cluster 0 pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:47.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:47 smithi189 bash[23614]: cluster 2024-03-07T03:04:45.336036+0000 mgr.y (mgr.14152) 299 : cluster 0 pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:49.530 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:49 smithi189 bash[23614]: cluster 2024-03-07T03:04:47.336630+0000 mgr.y (mgr.14152) 300 : cluster 0 pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:49.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:49 smithi145 bash[21339]: cluster 2024-03-07T03:04:47.336630+0000 mgr.y (mgr.14152) 300 : cluster 0 pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:49.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:49 smithi145 bash[30906]: cluster 2024-03-07T03:04:47.336630+0000 mgr.y (mgr.14152) 300 : cluster 0 pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:51.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:51 smithi145 bash[21339]: cluster 2024-03-07T03:04:49.337186+0000 mgr.y (mgr.14152) 301 : cluster 0 pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:51.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:51 smithi145 bash[30906]: cluster 2024-03-07T03:04:49.337186+0000 mgr.y (mgr.14152) 301 : cluster 0 pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:51.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:51 smithi189 bash[23614]: cluster 2024-03-07T03:04:49.337186+0000 mgr.y (mgr.14152) 301 : cluster 0 pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:52 smithi145 bash[21339]: audit 2024-03-07T03:04:51.924663+0000 mon.a (mon.0) 600 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-07T03:04:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:52 smithi145 bash[21339]: audit 2024-03-07T03:04:51.926449+0000 mon.a (mon.0) 601 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:52.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:52 smithi145 bash[30906]: audit 2024-03-07T03:04:51.924663+0000 mon.a (mon.0) 600 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-07T03:04:52.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:52 smithi145 bash[30906]: audit 2024-03-07T03:04:51.926449+0000 mon.a (mon.0) 601 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:52.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:52 smithi189 bash[23614]: audit 2024-03-07T03:04:51.924663+0000 mon.a (mon.0) 600 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-07T03:04:52.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:52 smithi189 bash[23614]: audit 2024-03-07T03:04:51.926449+0000 mon.a (mon.0) 601 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:53.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:53 smithi145 bash[21339]: cluster 2024-03-07T03:04:51.337729+0000 mgr.y (mgr.14152) 302 : cluster 0 pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:53.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:53 smithi145 bash[21339]: cephadm 2024-03-07T03:04:51.928053+0000 mgr.y (mgr.14152) 303 : cephadm 1 Deploying daemon osd.7 on smithi189 2024-03-07T03:04:53.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:53 smithi145 bash[30906]: cluster 2024-03-07T03:04:51.337729+0000 mgr.y (mgr.14152) 302 : cluster 0 pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:53.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:53 smithi145 bash[30906]: cephadm 2024-03-07T03:04:51.928053+0000 mgr.y (mgr.14152) 303 : cephadm 1 Deploying daemon osd.7 on smithi189 2024-03-07T03:04:53.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:53 smithi189 bash[23614]: cluster 2024-03-07T03:04:51.337729+0000 mgr.y (mgr.14152) 302 : cluster 0 pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:53.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:53 smithi189 bash[23614]: cephadm 2024-03-07T03:04:51.928053+0000 mgr.y (mgr.14152) 303 : cephadm 1 Deploying daemon osd.7 on smithi189 2024-03-07T03:04:55.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:55 smithi145 bash[21339]: cluster 2024-03-07T03:04:53.338338+0000 mgr.y (mgr.14152) 304 : cluster 0 pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:55.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:55 smithi145 bash[30906]: cluster 2024-03-07T03:04:53.338338+0000 mgr.y (mgr.14152) 304 : cluster 0 pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:55.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:55 smithi189 bash[23614]: cluster 2024-03-07T03:04:53.338338+0000 mgr.y (mgr.14152) 304 : cluster 0 pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:55.981 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:04:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:55.981 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:55.981 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:04:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:55.982 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:04:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:55.982 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:04:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:56.241 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:56.241 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:04:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:56.241 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:04:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:56.242 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:04:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:56.242 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:04:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:57.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:57 smithi189 bash[23614]: cluster 2024-03-07T03:04:55.338860+0000 mgr.y (mgr.14152) 305 : cluster 0 pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:57.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:57 smithi189 bash[23614]: audit 2024-03-07T03:04:56.299482+0000 mon.a (mon.0) 602 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:57.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:57 smithi189 bash[23614]: audit 2024-03-07T03:04:56.313391+0000 mon.a (mon.0) 603 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:57.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:57 smithi189 bash[23614]: audit 2024-03-07T03:04:56.326212+0000 mon.a (mon.0) 604 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:57.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:57 smithi145 bash[21339]: cluster 2024-03-07T03:04:55.338860+0000 mgr.y (mgr.14152) 305 : cluster 0 pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:57.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:57 smithi145 bash[21339]: audit 2024-03-07T03:04:56.299482+0000 mon.a (mon.0) 602 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:57.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:57 smithi145 bash[21339]: audit 2024-03-07T03:04:56.313391+0000 mon.a (mon.0) 603 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:57.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:57 smithi145 bash[21339]: audit 2024-03-07T03:04:56.326212+0000 mon.a (mon.0) 604 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:57.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:57 smithi145 bash[30906]: cluster 2024-03-07T03:04:55.338860+0000 mgr.y (mgr.14152) 305 : cluster 0 pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:57.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:57 smithi145 bash[30906]: audit 2024-03-07T03:04:56.299482+0000 mon.a (mon.0) 602 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:57.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:57 smithi145 bash[30906]: audit 2024-03-07T03:04:56.313391+0000 mon.a (mon.0) 603 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:04:57.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:57 smithi145 bash[30906]: audit 2024-03-07T03:04:56.326212+0000 mon.a (mon.0) 604 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:59.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:04:59 smithi145 bash[21339]: cluster 2024-03-07T03:04:57.339383+0000 mgr.y (mgr.14152) 306 : cluster 0 pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:59.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:04:59 smithi145 bash[30906]: cluster 2024-03-07T03:04:57.339383+0000 mgr.y (mgr.14152) 306 : cluster 0 pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:04:59.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:04:59 smithi189 bash[23614]: cluster 2024-03-07T03:04:57.339383+0000 mgr.y (mgr.14152) 306 : cluster 0 pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:05:00.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:00 smithi145 bash[21339]: cluster 2024-03-07T03:04:59.339964+0000 mgr.y (mgr.14152) 307 : cluster 0 pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:05:00.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:00 smithi145 bash[30906]: cluster 2024-03-07T03:04:59.339964+0000 mgr.y (mgr.14152) 307 : cluster 0 pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:05:00.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:00 smithi189 bash[23614]: cluster 2024-03-07T03:04:59.339964+0000 mgr.y (mgr.14152) 307 : cluster 0 pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:05:02.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:02 smithi189 bash[23614]: cluster 2024-03-07T03:05:01.340558+0000 mgr.y (mgr.14152) 308 : cluster 0 pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:05:02.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:02 smithi189 bash[23614]: audit 2024-03-07T03:05:01.575178+0000 mon.a (mon.0) 605 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:02.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:02 smithi189 bash[23614]: audit 2024-03-07T03:05:01.586030+0000 mon.a (mon.0) 606 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:02 smithi189 bash[23614]: audit 2024-03-07T03:05:01.588334+0000 mon.a (mon.0) 607 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:02 smithi189 bash[23614]: audit 2024-03-07T03:05:01.590322+0000 mon.a (mon.0) 608 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:02 smithi189 bash[23614]: audit 2024-03-07T03:05:01.601141+0000 mon.a (mon.0) 609 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:02 smithi189 bash[23614]: audit 2024-03-07T03:05:02.473131+0000 mon.b (mon.1) 24 : audit 1 from='osd.7 [v2:172.21.15.189:6824/452348965,v1:172.21.15.189:6825/452348965]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-07T03:05:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:02 smithi189 bash[23614]: audit 2024-03-07T03:05:02.474610+0000 mon.a (mon.0) 610 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-07T03:05:03.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[21339]: cluster 2024-03-07T03:05:01.340558+0000 mgr.y (mgr.14152) 308 : cluster 0 pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:05:03.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[21339]: audit 2024-03-07T03:05:01.575178+0000 mon.a (mon.0) 605 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:03.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[21339]: audit 2024-03-07T03:05:01.586030+0000 mon.a (mon.0) 606 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:03.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[21339]: audit 2024-03-07T03:05:01.588334+0000 mon.a (mon.0) 607 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:03.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[21339]: audit 2024-03-07T03:05:01.590322+0000 mon.a (mon.0) 608 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:03.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[21339]: audit 2024-03-07T03:05:01.601141+0000 mon.a (mon.0) 609 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:03.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[21339]: audit 2024-03-07T03:05:02.473131+0000 mon.b (mon.1) 24 : audit 1 from='osd.7 [v2:172.21.15.189:6824/452348965,v1:172.21.15.189:6825/452348965]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-07T03:05:03.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[21339]: audit 2024-03-07T03:05:02.474610+0000 mon.a (mon.0) 610 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-07T03:05:03.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[30906]: cluster 2024-03-07T03:05:01.340558+0000 mgr.y (mgr.14152) 308 : cluster 0 pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:05:03.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[30906]: audit 2024-03-07T03:05:01.575178+0000 mon.a (mon.0) 605 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:03.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[30906]: audit 2024-03-07T03:05:01.586030+0000 mon.a (mon.0) 606 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:03.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[30906]: audit 2024-03-07T03:05:01.588334+0000 mon.a (mon.0) 607 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:03.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[30906]: audit 2024-03-07T03:05:01.590322+0000 mon.a (mon.0) 608 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:03.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[30906]: audit 2024-03-07T03:05:01.601141+0000 mon.a (mon.0) 609 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:03.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[30906]: audit 2024-03-07T03:05:02.473131+0000 mon.b (mon.1) 24 : audit 1 from='osd.7 [v2:172.21.15.189:6824/452348965,v1:172.21.15.189:6825/452348965]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-07T03:05:03.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:02 smithi145 bash[30906]: audit 2024-03-07T03:05:02.474610+0000 mon.a (mon.0) 610 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-07T03:05:03.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:03 smithi189 bash[23614]: audit 2024-03-07T03:05:02.605923+0000 mon.a (mon.0) 611 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-07T03:05:03.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:03 smithi189 bash[23614]: cluster 2024-03-07T03:05:02.606172+0000 mon.a (mon.0) 612 : cluster 0 osdmap e53: 8 total, 7 up, 8 in 2024-03-07T03:05:03.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:03 smithi189 bash[23614]: audit 2024-03-07T03:05:02.606455+0000 mon.a (mon.0) 613 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:03.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:03 smithi189 bash[23614]: audit 2024-03-07T03:05:02.608413+0000 mon.b (mon.1) 25 : audit 1 from='osd.7 [v2:172.21.15.189:6824/452348965,v1:172.21.15.189:6825/452348965]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:05:03.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:03 smithi189 bash[23614]: audit 2024-03-07T03:05:02.609795+0000 mon.a (mon.0) 614 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:05:04.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:03 smithi145 bash[21339]: audit 2024-03-07T03:05:02.605923+0000 mon.a (mon.0) 611 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-07T03:05:04.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:03 smithi145 bash[21339]: cluster 2024-03-07T03:05:02.606172+0000 mon.a (mon.0) 612 : cluster 0 osdmap e53: 8 total, 7 up, 8 in 2024-03-07T03:05:04.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:03 smithi145 bash[21339]: audit 2024-03-07T03:05:02.606455+0000 mon.a (mon.0) 613 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:04.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:03 smithi145 bash[21339]: audit 2024-03-07T03:05:02.608413+0000 mon.b (mon.1) 25 : audit 1 from='osd.7 [v2:172.21.15.189:6824/452348965,v1:172.21.15.189:6825/452348965]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:05:04.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:03 smithi145 bash[21339]: audit 2024-03-07T03:05:02.609795+0000 mon.a (mon.0) 614 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:05:04.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:03 smithi145 bash[30906]: audit 2024-03-07T03:05:02.605923+0000 mon.a (mon.0) 611 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-07T03:05:04.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:03 smithi145 bash[30906]: cluster 2024-03-07T03:05:02.606172+0000 mon.a (mon.0) 612 : cluster 0 osdmap e53: 8 total, 7 up, 8 in 2024-03-07T03:05:04.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:03 smithi145 bash[30906]: audit 2024-03-07T03:05:02.606455+0000 mon.a (mon.0) 613 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:04.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:03 smithi145 bash[30906]: audit 2024-03-07T03:05:02.608413+0000 mon.b (mon.1) 25 : audit 1 from='osd.7 [v2:172.21.15.189:6824/452348965,v1:172.21.15.189:6825/452348965]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:05:04.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:03 smithi145 bash[30906]: audit 2024-03-07T03:05:02.609795+0000 mon.a (mon.0) 614 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:05:04.687 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 7 on host 'smithi189' 2024-03-07T03:05:04.906 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:04 smithi189 bash[23614]: cluster 2024-03-07T03:05:03.341198+0000 mgr.y (mgr.14152) 309 : cluster 0 pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:05:04.907 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:04 smithi189 bash[23614]: audit 2024-03-07T03:05:03.606983+0000 mon.a (mon.0) 615 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-07T03:05:04.907 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:04 smithi189 bash[23614]: cluster 2024-03-07T03:05:03.607118+0000 mon.a (mon.0) 616 : cluster 0 osdmap e54: 8 total, 7 up, 8 in 2024-03-07T03:05:04.907 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:04 smithi189 bash[23614]: audit 2024-03-07T03:05:03.607763+0000 mon.a (mon.0) 617 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:04.907 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:04 smithi189 bash[23614]: audit 2024-03-07T03:05:03.618810+0000 mon.a (mon.0) 618 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:05.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:04 smithi145 bash[21339]: cluster 2024-03-07T03:05:03.341198+0000 mgr.y (mgr.14152) 309 : cluster 0 pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:05:05.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:04 smithi145 bash[21339]: audit 2024-03-07T03:05:03.606983+0000 mon.a (mon.0) 615 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-07T03:05:05.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:04 smithi145 bash[21339]: cluster 2024-03-07T03:05:03.607118+0000 mon.a (mon.0) 616 : cluster 0 osdmap e54: 8 total, 7 up, 8 in 2024-03-07T03:05:05.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:04 smithi145 bash[21339]: audit 2024-03-07T03:05:03.607763+0000 mon.a (mon.0) 617 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:05.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:04 smithi145 bash[21339]: audit 2024-03-07T03:05:03.618810+0000 mon.a (mon.0) 618 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:05.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:04 smithi145 bash[30906]: cluster 2024-03-07T03:05:03.341198+0000 mgr.y (mgr.14152) 309 : cluster 0 pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:05:05.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:04 smithi145 bash[30906]: audit 2024-03-07T03:05:03.606983+0000 mon.a (mon.0) 615 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-03-07T03:05:05.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:04 smithi145 bash[30906]: cluster 2024-03-07T03:05:03.607118+0000 mon.a (mon.0) 616 : cluster 0 osdmap e54: 8 total, 7 up, 8 in 2024-03-07T03:05:05.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:04 smithi145 bash[30906]: audit 2024-03-07T03:05:03.607763+0000 mon.a (mon.0) 617 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:05.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:04 smithi145 bash[30906]: audit 2024-03-07T03:05:03.618810+0000 mon.a (mon.0) 618 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:05.306 DEBUG:teuthology.orchestra.run.smithi189:osd.7> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.7.service 2024-03-07T03:05:05.308 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-03-07T03:05:05.309 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd stat -f json 2024-03-07T03:05:06.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[21339]: cluster 2024-03-07T03:05:03.479764+0000 osd.7 (osd.7) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:05:06.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[21339]: cluster 2024-03-07T03:05:03.479898+0000 osd.7 (osd.7) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:05:06.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[21339]: audit 2024-03-07T03:05:04.615181+0000 mon.a (mon.0) 619 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:06.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[21339]: cluster 2024-03-07T03:05:04.622401+0000 mon.a (mon.0) 620 : cluster 1 osd.7 [v2:172.21.15.189:6824/452348965,v1:172.21.15.189:6825/452348965] boot 2024-03-07T03:05:06.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[21339]: cluster 2024-03-07T03:05:04.622504+0000 mon.a (mon.0) 621 : cluster 0 osdmap e55: 8 total, 8 up, 8 in 2024-03-07T03:05:06.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[21339]: audit 2024-03-07T03:05:04.623812+0000 mon.a (mon.0) 622 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:06.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[21339]: audit 2024-03-07T03:05:04.648909+0000 mon.a (mon.0) 623 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:06.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[21339]: audit 2024-03-07T03:05:04.654587+0000 mon.a (mon.0) 624 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:05:06.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[21339]: audit 2024-03-07T03:05:04.661001+0000 mon.a (mon.0) 625 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:06.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[30906]: cluster 2024-03-07T03:05:03.479764+0000 osd.7 (osd.7) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:05:06.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[30906]: cluster 2024-03-07T03:05:03.479898+0000 osd.7 (osd.7) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:05:06.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[30906]: audit 2024-03-07T03:05:04.615181+0000 mon.a (mon.0) 619 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:06.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[30906]: cluster 2024-03-07T03:05:04.622401+0000 mon.a (mon.0) 620 : cluster 1 osd.7 [v2:172.21.15.189:6824/452348965,v1:172.21.15.189:6825/452348965] boot 2024-03-07T03:05:06.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[30906]: cluster 2024-03-07T03:05:04.622504+0000 mon.a (mon.0) 621 : cluster 0 osdmap e55: 8 total, 8 up, 8 in 2024-03-07T03:05:06.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[30906]: audit 2024-03-07T03:05:04.623812+0000 mon.a (mon.0) 622 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:06.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[30906]: audit 2024-03-07T03:05:04.648909+0000 mon.a (mon.0) 623 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:06.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[30906]: audit 2024-03-07T03:05:04.654587+0000 mon.a (mon.0) 624 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:05:06.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:05 smithi145 bash[30906]: audit 2024-03-07T03:05:04.661001+0000 mon.a (mon.0) 625 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:06.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:05 smithi189 bash[23614]: cluster 2024-03-07T03:05:03.479764+0000 osd.7 (osd.7) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:05:06.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:05 smithi189 bash[23614]: cluster 2024-03-07T03:05:03.479898+0000 osd.7 (osd.7) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:05:06.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:05 smithi189 bash[23614]: audit 2024-03-07T03:05:04.615181+0000 mon.a (mon.0) 619 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:06.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:05 smithi189 bash[23614]: cluster 2024-03-07T03:05:04.622401+0000 mon.a (mon.0) 620 : cluster 1 osd.7 [v2:172.21.15.189:6824/452348965,v1:172.21.15.189:6825/452348965] boot 2024-03-07T03:05:06.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:05 smithi189 bash[23614]: cluster 2024-03-07T03:05:04.622504+0000 mon.a (mon.0) 621 : cluster 0 osdmap e55: 8 total, 8 up, 8 in 2024-03-07T03:05:06.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:05 smithi189 bash[23614]: audit 2024-03-07T03:05:04.623812+0000 mon.a (mon.0) 622 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:05:06.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:05 smithi189 bash[23614]: audit 2024-03-07T03:05:04.648909+0000 mon.a (mon.0) 623 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:06.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:05 smithi189 bash[23614]: audit 2024-03-07T03:05:04.654587+0000 mon.a (mon.0) 624 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:05:06.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:05 smithi189 bash[23614]: audit 2024-03-07T03:05:04.661001+0000 mon.a (mon.0) 625 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:07.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:06 smithi145 bash[30906]: cluster 2024-03-07T03:05:05.341883+0000 mgr.y (mgr.14152) 310 : cluster 0 pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:05:07.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:06 smithi145 bash[30906]: cluster 2024-03-07T03:05:05.629826+0000 mon.a (mon.0) 626 : cluster 0 osdmap e56: 8 total, 8 up, 8 in 2024-03-07T03:05:07.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:06 smithi145 bash[30906]: cluster 2024-03-07T03:05:06.630710+0000 mon.a (mon.0) 627 : cluster 0 osdmap e57: 8 total, 8 up, 8 in 2024-03-07T03:05:07.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:06 smithi145 bash[21339]: cluster 2024-03-07T03:05:05.341883+0000 mgr.y (mgr.14152) 310 : cluster 0 pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:05:07.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:06 smithi145 bash[21339]: cluster 2024-03-07T03:05:05.629826+0000 mon.a (mon.0) 626 : cluster 0 osdmap e56: 8 total, 8 up, 8 in 2024-03-07T03:05:07.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:06 smithi145 bash[21339]: cluster 2024-03-07T03:05:06.630710+0000 mon.a (mon.0) 627 : cluster 0 osdmap e57: 8 total, 8 up, 8 in 2024-03-07T03:05:07.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:06 smithi189 bash[23614]: cluster 2024-03-07T03:05:05.341883+0000 mgr.y (mgr.14152) 310 : cluster 0 pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:05:07.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:06 smithi189 bash[23614]: cluster 2024-03-07T03:05:05.629826+0000 mon.a (mon.0) 626 : cluster 0 osdmap e56: 8 total, 8 up, 8 in 2024-03-07T03:05:07.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:06 smithi189 bash[23614]: cluster 2024-03-07T03:05:06.630710+0000 mon.a (mon.0) 627 : cluster 0 osdmap e57: 8 total, 8 up, 8 in 2024-03-07T03:05:09.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:08 smithi145 bash[21339]: cluster 2024-03-07T03:05:07.342496+0000 mgr.y (mgr.14152) 311 : cluster 0 pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:09.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:08 smithi145 bash[21339]: cluster 2024-03-07T03:05:07.649474+0000 mon.a (mon.0) 628 : cluster 0 osdmap e58: 8 total, 8 up, 8 in 2024-03-07T03:05:09.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:08 smithi145 bash[30906]: cluster 2024-03-07T03:05:07.342496+0000 mgr.y (mgr.14152) 311 : cluster 0 pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:09.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:08 smithi145 bash[30906]: cluster 2024-03-07T03:05:07.649474+0000 mon.a (mon.0) 628 : cluster 0 osdmap e58: 8 total, 8 up, 8 in 2024-03-07T03:05:09.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:08 smithi189 bash[23614]: cluster 2024-03-07T03:05:07.342496+0000 mgr.y (mgr.14152) 311 : cluster 0 pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:09.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:08 smithi189 bash[23614]: cluster 2024-03-07T03:05:07.649474+0000 mon.a (mon.0) 628 : cluster 0 osdmap e58: 8 total, 8 up, 8 in 2024-03-07T03:05:09.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:09 smithi189 bash[23614]: cluster 2024-03-07T03:05:08.659019+0000 mon.a (mon.0) 629 : cluster 0 osdmap e59: 8 total, 8 up, 8 in 2024-03-07T03:05:10.045 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:05:10.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:09 smithi145 bash[21339]: cluster 2024-03-07T03:05:08.659019+0000 mon.a (mon.0) 629 : cluster 0 osdmap e59: 8 total, 8 up, 8 in 2024-03-07T03:05:10.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:09 smithi145 bash[30906]: cluster 2024-03-07T03:05:08.659019+0000 mon.a (mon.0) 629 : cluster 0 osdmap e59: 8 total, 8 up, 8 in 2024-03-07T03:05:11.025 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:10 smithi189 bash[23614]: cluster 2024-03-07T03:05:09.343024+0000 mgr.y (mgr.14152) 312 : cluster 0 pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:11.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:10 smithi145 bash[21339]: cluster 2024-03-07T03:05:09.343024+0000 mgr.y (mgr.14152) 312 : cluster 0 pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:11.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:10 smithi145 bash[30906]: cluster 2024-03-07T03:05:09.343024+0000 mgr.y (mgr.14152) 312 : cluster 0 pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:12.970 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:12 smithi189 bash[23614]: cluster 2024-03-07T03:05:11.343640+0000 mgr.y (mgr.14152) 313 : cluster 0 pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-03-07T03:05:13.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:12 smithi145 bash[30906]: cluster 2024-03-07T03:05:11.343640+0000 mgr.y (mgr.14152) 313 : cluster 0 pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-03-07T03:05:13.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:12 smithi145 bash[21339]: cluster 2024-03-07T03:05:11.343640+0000 mgr.y (mgr.14152) 313 : cluster 0 pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-03-07T03:05:13.169 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:05:13.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:13 smithi145 bash[21339]: audit 2024-03-07T03:05:13.173485+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.145:0/4151001173' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-07T03:05:13.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:13 smithi145 bash[21339]: audit 2024-03-07T03:05:13.654453+0000 mon.a (mon.0) 630 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:13.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:13 smithi145 bash[21339]: audit 2024-03-07T03:05:13.664848+0000 mon.a (mon.0) 631 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:13.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:13 smithi145 bash[21339]: audit 2024-03-07T03:05:13.667540+0000 mon.a (mon.0) 632 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:13.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:13 smithi145 bash[30906]: audit 2024-03-07T03:05:13.173485+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.145:0/4151001173' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-07T03:05:13.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:13 smithi145 bash[30906]: audit 2024-03-07T03:05:13.654453+0000 mon.a (mon.0) 630 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:13.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:13 smithi145 bash[30906]: audit 2024-03-07T03:05:13.664848+0000 mon.a (mon.0) 631 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:13.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:13 smithi145 bash[30906]: audit 2024-03-07T03:05:13.667540+0000 mon.a (mon.0) 632 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:13.954 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1709780704,"num_in_osds":8,"osd_in_since":1709780679,"num_remapped_pgs":0} 2024-03-07T03:05:13.955 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:05:14.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:13 smithi189 bash[23614]: audit 2024-03-07T03:05:13.173485+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.145:0/4151001173' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-07T03:05:14.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:13 smithi189 bash[23614]: audit 2024-03-07T03:05:13.654453+0000 mon.a (mon.0) 630 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:14.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:13 smithi189 bash[23614]: audit 2024-03-07T03:05:13.664848+0000 mon.a (mon.0) 631 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:14.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:13 smithi189 bash[23614]: audit 2024-03-07T03:05:13.667540+0000 mon.a (mon.0) 632 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:15.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[21339]: cluster 2024-03-07T03:05:13.344298+0000 mgr.y (mgr.14152) 314 : cluster 0 pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-07T03:05:15.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[21339]: cephadm 2024-03-07T03:05:13.644556+0000 mgr.y (mgr.14152) 315 : cephadm 1 Detected new or changed devices on smithi189 2024-03-07T03:05:15.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[21339]: audit 2024-03-07T03:05:13.669806+0000 mon.a (mon.0) 633 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:15.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[21339]: audit 2024-03-07T03:05:13.671628+0000 mon.a (mon.0) 634 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:15.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[21339]: audit 2024-03-07T03:05:13.681331+0000 mon.a (mon.0) 635 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:15.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[21339]: cephadm 2024-03-07T03:05:13.682607+0000 mgr.y (mgr.14152) 316 : cephadm 1 Adjusting osd_memory_target on smithi189 to 4316M 2024-03-07T03:05:15.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[21339]: audit 2024-03-07T03:05:13.691054+0000 mon.a (mon.0) 636 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:15.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[21339]: audit 2024-03-07T03:05:13.692999+0000 mon.a (mon.0) 637 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:15.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[21339]: audit 2024-03-07T03:05:13.694748+0000 mon.a (mon.0) 638 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:15.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[21339]: audit 2024-03-07T03:05:13.704693+0000 mon.a (mon.0) 639 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:15.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[30906]: cluster 2024-03-07T03:05:13.344298+0000 mgr.y (mgr.14152) 314 : cluster 0 pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-07T03:05:15.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[30906]: cephadm 2024-03-07T03:05:13.644556+0000 mgr.y (mgr.14152) 315 : cephadm 1 Detected new or changed devices on smithi189 2024-03-07T03:05:15.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[30906]: audit 2024-03-07T03:05:13.669806+0000 mon.a (mon.0) 633 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:15.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[30906]: audit 2024-03-07T03:05:13.671628+0000 mon.a (mon.0) 634 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:15.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[30906]: audit 2024-03-07T03:05:13.681331+0000 mon.a (mon.0) 635 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:15.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[30906]: cephadm 2024-03-07T03:05:13.682607+0000 mgr.y (mgr.14152) 316 : cephadm 1 Adjusting osd_memory_target on smithi189 to 4316M 2024-03-07T03:05:15.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[30906]: audit 2024-03-07T03:05:13.691054+0000 mon.a (mon.0) 636 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:15.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[30906]: audit 2024-03-07T03:05:13.692999+0000 mon.a (mon.0) 637 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:15.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[30906]: audit 2024-03-07T03:05:13.694748+0000 mon.a (mon.0) 638 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:15.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:14 smithi145 bash[30906]: audit 2024-03-07T03:05:13.704693+0000 mon.a (mon.0) 639 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:15.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:14 smithi189 bash[23614]: cluster 2024-03-07T03:05:13.344298+0000 mgr.y (mgr.14152) 314 : cluster 0 pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-07T03:05:15.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:14 smithi189 bash[23614]: cephadm 2024-03-07T03:05:13.644556+0000 mgr.y (mgr.14152) 315 : cephadm 1 Detected new or changed devices on smithi189 2024-03-07T03:05:15.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:14 smithi189 bash[23614]: audit 2024-03-07T03:05:13.669806+0000 mon.a (mon.0) 633 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:15.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:14 smithi189 bash[23614]: audit 2024-03-07T03:05:13.671628+0000 mon.a (mon.0) 634 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:15.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:14 smithi189 bash[23614]: audit 2024-03-07T03:05:13.681331+0000 mon.a (mon.0) 635 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:15.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:14 smithi189 bash[23614]: cephadm 2024-03-07T03:05:13.682607+0000 mgr.y (mgr.14152) 316 : cephadm 1 Adjusting osd_memory_target on smithi189 to 4316M 2024-03-07T03:05:15.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:14 smithi189 bash[23614]: audit 2024-03-07T03:05:13.691054+0000 mon.a (mon.0) 636 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:15.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:14 smithi189 bash[23614]: audit 2024-03-07T03:05:13.692999+0000 mon.a (mon.0) 637 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:15.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:14 smithi189 bash[23614]: audit 2024-03-07T03:05:13.694748+0000 mon.a (mon.0) 638 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:15.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:14 smithi189 bash[23614]: audit 2024-03-07T03:05:13.704693+0000 mon.a (mon.0) 639 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:17.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:16 smithi145 bash[21339]: cluster 2024-03-07T03:05:15.344903+0000 mgr.y (mgr.14152) 317 : cluster 0 pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-07T03:05:17.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:16 smithi145 bash[30906]: cluster 2024-03-07T03:05:15.344903+0000 mgr.y (mgr.14152) 317 : cluster 0 pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-07T03:05:17.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:16 smithi189 bash[23614]: cluster 2024-03-07T03:05:15.344903+0000 mgr.y (mgr.14152) 317 : cluster 0 pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-07T03:05:18.666 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:05:19.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:19 smithi145 bash[21339]: cluster 2024-03-07T03:05:17.345518+0000 mgr.y (mgr.14152) 318 : cluster 0 pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-03-07T03:05:19.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:19 smithi145 bash[30906]: cluster 2024-03-07T03:05:17.345518+0000 mgr.y (mgr.14152) 318 : cluster 0 pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-03-07T03:05:19.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:19 smithi189 bash[23614]: cluster 2024-03-07T03:05:17.345518+0000 mgr.y (mgr.14152) 318 : cluster 0 pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-03-07T03:05:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:21 smithi145 bash[21339]: cluster 2024-03-07T03:05:19.346105+0000 mgr.y (mgr.14152) 319 : cluster 0 pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-03-07T03:05:21.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:21 smithi145 bash[30906]: cluster 2024-03-07T03:05:19.346105+0000 mgr.y (mgr.14152) 319 : cluster 0 pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-03-07T03:05:21.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:21 smithi189 bash[23614]: cluster 2024-03-07T03:05:19.346105+0000 mgr.y (mgr.14152) 319 : cluster 0 pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-03-07T03:05:21.866 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:05:21.866 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":59,"fsid":"8947a0f8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:55:42.039360+0000","modified":"2024-03-07T03:05:08.644813+0000","last_up_change":"2024-03-07T03:05:04.610834+0000","last_in_change":"2024-03-07T03:04:39.932769+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:00:59.846304+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"242c63ee-0417-4ebf-bd71-6239f5d3634f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6803","nonce":1367316946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6805","nonce":1367316946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6809","nonce":1367316946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6807","nonce":1367316946}]},"public_addr":"172.21.15.145:6803/1367316946","cluster_addr":"172.21.15.145:6805/1367316946","heartbeat_back_addr":"172.21.15.145:6809/1367316946","heartbeat_front_addr":"172.21.15.145:6807/1367316946","state":["exists","up"]},{"osd":1,"uuid":"3c2818da-5d32-4f7a-8a63-8d7d679961e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6811","nonce":1873038359}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6813","nonce":1873038359}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6817","nonce":1873038359}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6815","nonce":1873038359}]},"public_addr":"172.21.15.145:6811/1873038359","cluster_addr":"172.21.15.145:6813/1873038359","heartbeat_back_addr":"172.21.15.145:6817/1873038359","heartbeat_front_addr":"172.21.15.145:6815/1873038359","state":["exists","up"]},{"osd":2,"uuid":"905ec219-dd9c-4f30-bdab-ed78d1405c11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6819","nonce":462216015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6821","nonce":462216015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6825","nonce":462216015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6823","nonce":462216015}]},"public_addr":"172.21.15.145:6819/462216015","cluster_addr":"172.21.15.145:6821/462216015","heartbeat_back_addr":"172.21.15.145:6825/462216015","heartbeat_front_addr":"172.21.15.145:6823/462216015","state":["exists","up"]},{"osd":3,"uuid":"5f466751-f5a7-4d18-87ab-96251b77b663","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6827","nonce":2306759084}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6829","nonce":2306759084}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6833","nonce":2306759084}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6831","nonce":2306759084}]},"public_addr":"172.21.15.145:6827/2306759084","cluster_addr":"172.21.15.145:6829/2306759084","heartbeat_back_addr":"172.21.15.145:6833/2306759084","heartbeat_front_addr":"172.21.15.145:6831/2306759084","state":["exists","up"]},{"osd":4,"uuid":"633f56af-3ff8-4954-97b4-bc8122bb10ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6801","nonce":3152992669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6803","nonce":3152992669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6807","nonce":3152992669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6805","nonce":3152992669}]},"public_addr":"172.21.15.189:6801/3152992669","cluster_addr":"172.21.15.189:6803/3152992669","heartbeat_back_addr":"172.21.15.189:6807/3152992669","heartbeat_front_addr":"172.21.15.189:6805/3152992669","state":["exists","up"]},{"osd":5,"uuid":"531d5127-60fe-4669-ab7d-fe4f5e5753cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6809","nonce":2096444201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6811","nonce":2096444201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6815","nonce":2096444201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6813","nonce":2096444201}]},"public_addr":"172.21.15.189:6809/2096444201","cluster_addr":"172.21.15.189:6811/2096444201","heartbeat_back_addr":"172.21.15.189:6815/2096444201","heartbeat_front_addr":"172.21.15.189:6813/2096444201","state":["exists","up"]},{"osd":6,"uuid":"8f2b456e-712e-45f1-8d84-0fdc4a6d860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6817","nonce":4219543665}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6819","nonce":4219543665}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6823","nonce":4219543665}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6821","nonce":4219543665}]},"public_addr":"172.21.15.189:6817/4219543665","cluster_addr":"172.21.15.189:6819/4219543665","heartbeat_back_addr":"172.21.15.189:6823/4219543665","heartbeat_front_addr":"172.21.15.189:6821/4219543665","state":["exists","up"]},{"osd":7,"uuid":"24dc7eba-4cd4-414d-93d2-598f5f2e655c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6825","nonce":452348965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6827","nonce":452348965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6831","nonce":452348965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6829","nonce":452348965}]},"public_addr":"172.21.15.189:6825/452348965","cluster_addr":"172.21.15.189:6827/452348965","heartbeat_back_addr":"172.21.15.189:6831/452348965","heartbeat_front_addr":"172.21.15.189:6829/452348965","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T02:59:11.112897+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:03.501748+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:56.635004+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:01:50.372735+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:38.150721+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:25.965239+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:14.345171+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:03.479901+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.145:0/2939737297":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/2236140381":"2024-03-08T02:56:14.659813+0000","172.21.15.145:6800/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/2696605014":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/131277059":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/2414597158":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:6800/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/321164949":"2024-03-08T02:56:14.659813+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:05:22.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:22 smithi145 bash[21339]: audit 2024-03-07T03:05:21.868461+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.145:0/2332401492' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:05:22.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:22 smithi145 bash[30906]: audit 2024-03-07T03:05:21.868461+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.145:0/2332401492' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:05:22.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:22 smithi189 bash[23614]: audit 2024-03-07T03:05:21.868461+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.145:0/2332401492' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:05:22.648 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-07T03:00:59.846304+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 7.889999866485596, 'score_stable': 7.889999866485596, 'optimal_score': 0.3799999952316284, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-07T03:05:22.648 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd pool get .mgr pg_num 2024-03-07T03:05:23.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:23 smithi145 bash[21339]: cluster 2024-03-07T03:05:21.346677+0000 mgr.y (mgr.14152) 320 : cluster 0 pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:05:23.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:23 smithi145 bash[30906]: cluster 2024-03-07T03:05:21.346677+0000 mgr.y (mgr.14152) 320 : cluster 0 pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:05:23.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:23 smithi189 bash[23614]: cluster 2024-03-07T03:05:21.346677+0000 mgr.y (mgr.14152) 320 : cluster 0 pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:05:25.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:25 smithi145 bash[21339]: cluster 2024-03-07T03:05:23.347279+0000 mgr.y (mgr.14152) 321 : cluster 0 pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:25.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:25 smithi145 bash[30906]: cluster 2024-03-07T03:05:23.347279+0000 mgr.y (mgr.14152) 321 : cluster 0 pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:25.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:25 smithi189 bash[23614]: cluster 2024-03-07T03:05:23.347279+0000 mgr.y (mgr.14152) 321 : cluster 0 pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:27.359 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:05:27.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:27 smithi145 bash[21339]: cluster 2024-03-07T03:05:25.347696+0000 mgr.y (mgr.14152) 322 : cluster 0 pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:27.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:27 smithi145 bash[30906]: cluster 2024-03-07T03:05:25.347696+0000 mgr.y (mgr.14152) 322 : cluster 0 pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:27.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:27 smithi189 bash[23614]: cluster 2024-03-07T03:05:25.347696+0000 mgr.y (mgr.14152) 322 : cluster 0 pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:29.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:29 smithi145 bash[21339]: cluster 2024-03-07T03:05:27.348217+0000 mgr.y (mgr.14152) 323 : cluster 0 pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:29.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:29 smithi145 bash[30906]: cluster 2024-03-07T03:05:27.348217+0000 mgr.y (mgr.14152) 323 : cluster 0 pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:29.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:29 smithi189 bash[23614]: cluster 2024-03-07T03:05:27.348217+0000 mgr.y (mgr.14152) 323 : cluster 0 pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:30.526 INFO:teuthology.orchestra.run.smithi145.stdout:pg_num: 1 2024-03-07T03:05:31.322 INFO:tasks.cephadm:Adding prometheus.a on smithi189 2024-03-07T03:05:31.322 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch apply prometheus '1;smithi189=a' 2024-03-07T03:05:31.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:31 smithi145 bash[30906]: cluster 2024-03-07T03:05:29.348791+0000 mgr.y (mgr.14152) 324 : cluster 0 pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:31.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:31 smithi145 bash[30906]: audit 2024-03-07T03:05:30.529453+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.145:0/1040926572' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-07T03:05:31.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:31 smithi145 bash[21339]: cluster 2024-03-07T03:05:29.348791+0000 mgr.y (mgr.14152) 324 : cluster 0 pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:31.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:31 smithi145 bash[21339]: audit 2024-03-07T03:05:30.529453+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.145:0/1040926572' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-07T03:05:31.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:31 smithi189 bash[23614]: cluster 2024-03-07T03:05:29.348791+0000 mgr.y (mgr.14152) 324 : cluster 0 pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:31.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:31 smithi189 bash[23614]: audit 2024-03-07T03:05:30.529453+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.145:0/1040926572' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-07T03:05:33.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:33 smithi145 bash[30906]: cluster 2024-03-07T03:05:31.349395+0000 mgr.y (mgr.14152) 325 : cluster 0 pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:33.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:33 smithi145 bash[21339]: cluster 2024-03-07T03:05:31.349395+0000 mgr.y (mgr.14152) 325 : cluster 0 pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:33.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:33 smithi189 bash[23614]: cluster 2024-03-07T03:05:31.349395+0000 mgr.y (mgr.14152) 325 : cluster 0 pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:35.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:35 smithi145 bash[21339]: cluster 2024-03-07T03:05:33.349970+0000 mgr.y (mgr.14152) 326 : cluster 0 pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:35.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:35 smithi145 bash[30906]: cluster 2024-03-07T03:05:33.349970+0000 mgr.y (mgr.14152) 326 : cluster 0 pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:35.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:35 smithi189 bash[23614]: cluster 2024-03-07T03:05:33.349970+0000 mgr.y (mgr.14152) 326 : cluster 0 pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:36.026 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:05:37.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:37 smithi145 bash[21339]: cluster 2024-03-07T03:05:35.350566+0000 mgr.y (mgr.14152) 327 : cluster 0 pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:37.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:37 smithi145 bash[30906]: cluster 2024-03-07T03:05:35.350566+0000 mgr.y (mgr.14152) 327 : cluster 0 pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:37.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:37 smithi189 bash[23614]: cluster 2024-03-07T03:05:35.350566+0000 mgr.y (mgr.14152) 327 : cluster 0 pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:38.532 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled prometheus update... 2024-03-07T03:05:39.098 DEBUG:teuthology.orchestra.run.smithi189:prometheus.a> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@prometheus.a.service 2024-03-07T03:05:39.101 INFO:tasks.cephadm:Adding node-exporter.a on smithi145 2024-03-07T03:05:39.101 INFO:tasks.cephadm:Adding node-exporter.b on smithi189 2024-03-07T03:05:39.101 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch apply node-exporter '2;smithi145=a;smithi189=b' 2024-03-07T03:05:39.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:39 smithi145 bash[30906]: cluster 2024-03-07T03:05:37.351185+0000 mgr.y (mgr.14152) 328 : cluster 0 pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:39.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:39 smithi145 bash[30906]: audit 2024-03-07T03:05:38.535661+0000 mon.a (mon.0) 640 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:39.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:39 smithi145 bash[30906]: audit 2024-03-07T03:05:38.552774+0000 mon.a (mon.0) 641 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:05:39.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:39 smithi145 bash[30906]: audit 2024-03-07T03:05:39.187132+0000 mon.a (mon.0) 642 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:39.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:39 smithi145 bash[30906]: audit 2024-03-07T03:05:39.188593+0000 mon.a (mon.0) 643 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:39.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:39 smithi145 bash[30906]: audit 2024-03-07T03:05:39.195784+0000 mon.a (mon.0) 644 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:39.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:39 smithi145 bash[21339]: cluster 2024-03-07T03:05:37.351185+0000 mgr.y (mgr.14152) 328 : cluster 0 pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:39.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:39 smithi145 bash[21339]: audit 2024-03-07T03:05:38.535661+0000 mon.a (mon.0) 640 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:39.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:39 smithi145 bash[21339]: audit 2024-03-07T03:05:38.552774+0000 mon.a (mon.0) 641 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:05:39.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:39 smithi145 bash[21339]: audit 2024-03-07T03:05:39.187132+0000 mon.a (mon.0) 642 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:39.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:39 smithi145 bash[21339]: audit 2024-03-07T03:05:39.188593+0000 mon.a (mon.0) 643 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:39.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:39 smithi145 bash[21339]: audit 2024-03-07T03:05:39.195784+0000 mon.a (mon.0) 644 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:39.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:39 smithi189 bash[23614]: cluster 2024-03-07T03:05:37.351185+0000 mgr.y (mgr.14152) 328 : cluster 0 pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:39.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:39 smithi189 bash[23614]: audit 2024-03-07T03:05:38.535661+0000 mon.a (mon.0) 640 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:39.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:39 smithi189 bash[23614]: audit 2024-03-07T03:05:38.552774+0000 mon.a (mon.0) 641 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:05:39.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:39 smithi189 bash[23614]: audit 2024-03-07T03:05:39.187132+0000 mon.a (mon.0) 642 : audit 0 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:39.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:39 smithi189 bash[23614]: audit 2024-03-07T03:05:39.188593+0000 mon.a (mon.0) 643 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:39.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:39 smithi189 bash[23614]: audit 2024-03-07T03:05:39.195784+0000 mon.a (mon.0) 644 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:40.088 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:39 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:40.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:40 smithi145 bash[30906]: audit 2024-03-07T03:05:38.524260+0000 mgr.y (mgr.14152) 329 : audit 0 from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:05:40.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:40 smithi145 bash[30906]: cephadm 2024-03-07T03:05:38.527683+0000 mgr.y (mgr.14152) 330 : cephadm 1 Saving service prometheus spec with placement smithi189=a;count:1 2024-03-07T03:05:40.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:40 smithi145 bash[21339]: audit 2024-03-07T03:05:38.524260+0000 mgr.y (mgr.14152) 329 : audit 0 from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:05:40.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:40 smithi145 bash[21339]: cephadm 2024-03-07T03:05:38.527683+0000 mgr.y (mgr.14152) 330 : cephadm 1 Saving service prometheus spec with placement smithi189=a;count:1 2024-03-07T03:05:40.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:40 smithi189 bash[23614]: audit 2024-03-07T03:05:38.524260+0000 mgr.y (mgr.14152) 329 : audit 0 from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:05:40.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:40 smithi189 bash[23614]: cephadm 2024-03-07T03:05:38.527683+0000 mgr.y (mgr.14152) 330 : cephadm 1 Saving service prometheus spec with placement smithi189=a;count:1 2024-03-07T03:05:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:41 smithi145 bash[30906]: cluster 2024-03-07T03:05:39.351556+0000 mgr.y (mgr.14152) 331 : cluster 0 pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:41.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:41 smithi145 bash[30906]: cephadm 2024-03-07T03:05:39.471260+0000 mgr.y (mgr.14152) 332 : cephadm 1 Deploying daemon prometheus.a on smithi189 2024-03-07T03:05:41.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:41 smithi145 bash[21339]: cluster 2024-03-07T03:05:39.351556+0000 mgr.y (mgr.14152) 331 : cluster 0 pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:41.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:41 smithi145 bash[21339]: cephadm 2024-03-07T03:05:39.471260+0000 mgr.y (mgr.14152) 332 : cephadm 1 Deploying daemon prometheus.a on smithi189 2024-03-07T03:05:41.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:41 smithi189 bash[23614]: cluster 2024-03-07T03:05:39.351556+0000 mgr.y (mgr.14152) 331 : cluster 0 pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:41.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:41 smithi189 bash[23614]: cephadm 2024-03-07T03:05:39.471260+0000 mgr.y (mgr.14152) 332 : cephadm 1 Deploying daemon prometheus.a on smithi189 2024-03-07T03:05:42.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:42 smithi189 bash[23614]: cluster 2024-03-07T03:05:41.351973+0000 mgr.y (mgr.14152) 333 : cluster 0 pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:42.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:42 smithi145 bash[30906]: cluster 2024-03-07T03:05:41.351973+0000 mgr.y (mgr.14152) 333 : cluster 0 pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:42.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:42 smithi145 bash[21339]: cluster 2024-03-07T03:05:41.351973+0000 mgr.y (mgr.14152) 333 : cluster 0 pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:43.827 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:05:45.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:45 smithi145 bash[30906]: cluster 2024-03-07T03:05:43.352524+0000 mgr.y (mgr.14152) 334 : cluster 0 pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:45.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:45 smithi145 bash[21339]: cluster 2024-03-07T03:05:43.352524+0000 mgr.y (mgr.14152) 334 : cluster 0 pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:45.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:45 smithi189 bash[23614]: cluster 2024-03-07T03:05:43.352524+0000 mgr.y (mgr.14152) 334 : cluster 0 pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:47.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:47 smithi145 bash[30906]: cluster 2024-03-07T03:05:45.353134+0000 mgr.y (mgr.14152) 335 : cluster 0 pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:47.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:47 smithi145 bash[21339]: cluster 2024-03-07T03:05:45.353134+0000 mgr.y (mgr.14152) 335 : cluster 0 pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:47.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:47 smithi189 bash[23614]: cluster 2024-03-07T03:05:45.353134+0000 mgr.y (mgr.14152) 335 : cluster 0 pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:49.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:49 smithi189 bash[23614]: cluster 2024-03-07T03:05:47.353709+0000 mgr.y (mgr.14152) 336 : cluster 0 pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:49.475 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled node-exporter update... 2024-03-07T03:05:49.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:49 smithi145 bash[21339]: cluster 2024-03-07T03:05:47.353709+0000 mgr.y (mgr.14152) 336 : cluster 0 pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:49.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:49 smithi145 bash[30906]: cluster 2024-03-07T03:05:47.353709+0000 mgr.y (mgr.14152) 336 : cluster 0 pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:50.352 DEBUG:teuthology.orchestra.run.smithi145:node-exporter.a> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@node-exporter.a.service 2024-03-07T03:05:50.354 DEBUG:teuthology.orchestra.run.smithi189:node-exporter.b> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@node-exporter.b.service 2024-03-07T03:05:50.357 INFO:tasks.cephadm:Adding alertmanager.a on smithi145 2024-03-07T03:05:50.357 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch apply alertmanager '1;smithi145=a' 2024-03-07T03:05:50.645 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:50 smithi189 bash[23614]: cluster 2024-03-07T03:05:49.354289+0000 mgr.y (mgr.14152) 337 : cluster 0 pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:50.645 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:50 smithi189 bash[23614]: audit 2024-03-07T03:05:49.465362+0000 mgr.y (mgr.14152) 338 : audit 0 from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi145=a;smithi189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:05:50.645 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:50 smithi189 bash[23614]: cephadm 2024-03-07T03:05:49.469024+0000 mgr.y (mgr.14152) 339 : cephadm 1 Saving service node-exporter spec with placement smithi145=a;smithi189=b;count:2 2024-03-07T03:05:50.645 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:50 smithi189 bash[23614]: audit 2024-03-07T03:05:49.478360+0000 mon.a (mon.0) 645 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:50.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:50 smithi145 bash[21339]: cluster 2024-03-07T03:05:49.354289+0000 mgr.y (mgr.14152) 337 : cluster 0 pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:50.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:50 smithi145 bash[21339]: audit 2024-03-07T03:05:49.465362+0000 mgr.y (mgr.14152) 338 : audit 0 from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi145=a;smithi189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:05:50.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:50 smithi145 bash[21339]: cephadm 2024-03-07T03:05:49.469024+0000 mgr.y (mgr.14152) 339 : cephadm 1 Saving service node-exporter spec with placement smithi145=a;smithi189=b;count:2 2024-03-07T03:05:50.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:50 smithi145 bash[21339]: audit 2024-03-07T03:05:49.478360+0000 mon.a (mon.0) 645 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:50 smithi145 bash[30906]: cluster 2024-03-07T03:05:49.354289+0000 mgr.y (mgr.14152) 337 : cluster 0 pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:50 smithi145 bash[30906]: audit 2024-03-07T03:05:49.465362+0000 mgr.y (mgr.14152) 338 : audit 0 from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi145=a;smithi189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:05:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:50 smithi145 bash[30906]: cephadm 2024-03-07T03:05:49.469024+0000 mgr.y (mgr.14152) 339 : cephadm 1 Saving service node-exporter spec with placement smithi145=a;smithi189=b;count:2 2024-03-07T03:05:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:50 smithi145 bash[30906]: audit 2024-03-07T03:05:49.478360+0000 mon.a (mon.0) 645 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:52.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:52 smithi189 bash[23614]: cluster 2024-03-07T03:05:51.354926+0000 mgr.y (mgr.14152) 340 : cluster 0 pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:52.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:52 smithi145 bash[21339]: cluster 2024-03-07T03:05:51.354926+0000 mgr.y (mgr.14152) 340 : cluster 0 pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:52.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:52 smithi145 bash[30906]: cluster 2024-03-07T03:05:51.354926+0000 mgr.y (mgr.14152) 340 : cluster 0 pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:05:52.978 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:52 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:52.979 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:05:52 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:52.979 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:05:52 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:52.979 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:05:52 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:52.979 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:05:52 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:52.980 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:52 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:52.980 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:52 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:52.980 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:52 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:52.980 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:52 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:53.338 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:53 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:53.338 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:53 smithi189 systemd[1]: Started Ceph prometheus.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:05:53.338 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:53 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:53.339 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:05:53 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:53.339 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:05:53 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:53.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:53 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:53.339 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:05:53 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:53.339 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:05:53 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:54.326 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[23614]: audit 2024-03-07T03:05:53.250812+0000 mon.a (mon.0) 646 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:54.326 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[23614]: audit 2024-03-07T03:05:53.260998+0000 mon.a (mon.0) 647 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:54.326 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[23614]: audit 2024-03-07T03:05:53.288449+0000 mon.a (mon.0) 648 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:54.326 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[23614]: audit 2024-03-07T03:05:53.298382+0000 mon.a (mon.0) 649 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-07T03:05:54.326 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.149Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-07T03:05:54.326 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.149Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.149Z caller=main.go:570 level=info host_details="(Linux 5.15.0-97-generic #107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024 x86_64 smithi189 (none))" 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.149Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.149Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.151Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.151Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.152Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.152Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.156Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.156Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.518µs 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.156Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.156Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.156Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.462µs wal_replay_duration=400.262µs wbl_replay_duration=108ns total_replay_duration=445.241µs 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.157Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-07T03:05:54.327 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.157Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-07T03:05:54.328 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.157Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-07T03:05:54.328 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.176Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.366602ms db_storage=1.642µs remote_storage=1.564µs web_handler=669ns query_engine=732ns scrape=278.6µs scrape_sd=99.836µs notify=1.011µs notify_sd=1.298µs rules=17.734424ms tracing=5.579µs 2024-03-07T03:05:54.328 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.176Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-07T03:05:54.328 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[47817]: ts=2024-03-07T03:05:54.176Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-07T03:05:54.497 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:54 smithi145 bash[21339]: audit 2024-03-07T03:05:53.250812+0000 mon.a (mon.0) 646 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:54.497 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:54 smithi145 bash[21339]: audit 2024-03-07T03:05:53.260998+0000 mon.a (mon.0) 647 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:54.497 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:54 smithi145 bash[21339]: audit 2024-03-07T03:05:53.288449+0000 mon.a (mon.0) 648 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:54.497 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:54 smithi145 bash[21339]: audit 2024-03-07T03:05:53.298382+0000 mon.a (mon.0) 649 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-07T03:05:54.497 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:54 smithi145 bash[21623]: ignoring --setuser ceph since I am not root 2024-03-07T03:05:54.497 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:54 smithi145 bash[21623]: ignoring --setgroup ceph since I am not root 2024-03-07T03:05:54.497 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:54 smithi145 bash[21623]: debug 2024-03-07T03:05:54.494+0000 7f288a953200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:05:54.498 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:54 smithi145 bash[30906]: audit 2024-03-07T03:05:53.250812+0000 mon.a (mon.0) 646 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:54.498 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:54 smithi145 bash[30906]: audit 2024-03-07T03:05:53.260998+0000 mon.a (mon.0) 647 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:54.498 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:54 smithi145 bash[30906]: audit 2024-03-07T03:05:53.288449+0000 mon.a (mon.0) 648 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' 2024-03-07T03:05:54.498 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:54 smithi145 bash[30906]: audit 2024-03-07T03:05:53.298382+0000 mon.a (mon.0) 649 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-07T03:05:54.588 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[24563]: ignoring --setuser ceph since I am not root 2024-03-07T03:05:54.588 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[24563]: ignoring --setgroup ceph since I am not root 2024-03-07T03:05:54.588 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[24563]: debug 2024-03-07T03:05:54.364+0000 7f4f8bfec700 1 -- 172.21.15.189:0/1776299067 <== mon.0 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5555667611e0 con 0x5555674e2800 2024-03-07T03:05:54.588 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:54 smithi189 bash[24563]: debug 2024-03-07T03:05:54.512+0000 7f4f95d30200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:05:55.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:55 smithi145 bash[21339]: audit 2024-03-07T03:05:54.303510+0000 mon.a (mon.0) 650 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-07T03:05:55.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:05:55 smithi145 bash[21339]: cluster 2024-03-07T03:05:54.303616+0000 mon.a (mon.0) 651 : cluster 0 mgrmap e16: y(active, since 9m), standbys: x 2024-03-07T03:05:55.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:55 smithi145 bash[30906]: audit 2024-03-07T03:05:54.303510+0000 mon.a (mon.0) 650 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-07T03:05:55.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:05:55 smithi145 bash[30906]: cluster 2024-03-07T03:05:54.303616+0000 mon.a (mon.0) 651 : cluster 0 mgrmap e16: y(active, since 9m), standbys: x 2024-03-07T03:05:55.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:55 smithi189 bash[23614]: audit 2024-03-07T03:05:54.303510+0000 mon.a (mon.0) 650 : audit 1 from='mgr.14152 172.21.15.145:0/680996607' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-07T03:05:55.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:05:55 smithi189 bash[23614]: cluster 2024-03-07T03:05:54.303616+0000 mon.a (mon.0) 651 : cluster 0 mgrmap e16: y(active, since 9m), standbys: x 2024-03-07T03:05:56.111 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:05:56.317 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:56 smithi145 bash[21623]: debug 2024-03-07T03:05:56.038+0000 7f288a953200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:05:56.318 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:56 smithi145 bash[21623]: debug 2024-03-07T03:05:56.134+0000 7f288a953200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:05:56.443 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:56 smithi189 bash[24563]: debug 2024-03-07T03:05:56.096+0000 7f4f95d30200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:05:56.443 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:56 smithi189 bash[24563]: debug 2024-03-07T03:05:56.196+0000 7f4f95d30200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:05:56.443 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:56 smithi189 bash[24563]: debug 2024-03-07T03:05:56.444+0000 7f4f95d30200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:05:56.817 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:56 smithi145 bash[21623]: debug 2024-03-07T03:05:56.382+0000 7f288a953200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:05:57.338 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:57 smithi189 bash[24563]: debug 2024-03-07T03:05:57.240+0000 7f4f95d30200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:05:57.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:57 smithi145 bash[21623]: debug 2024-03-07T03:05:57.162+0000 7f288a953200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:05:57.568 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:57 smithi145 bash[21623]: debug 2024-03-07T03:05:57.274+0000 7f288a953200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:05:57.568 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:57 smithi145 bash[21623]: debug 2024-03-07T03:05:57.378+0000 7f288a953200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:05:57.809 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:57 smithi189 bash[24563]: debug 2024-03-07T03:05:57.352+0000 7f4f95d30200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:05:57.809 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:57 smithi189 bash[24563]: debug 2024-03-07T03:05:57.456+0000 7f4f95d30200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:05:58.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:57 smithi145 bash[21623]: debug 2024-03-07T03:05:57.730+0000 7f288a953200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:05:58.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:57 smithi145 bash[21623]: debug 2024-03-07T03:05:57.846+0000 7f288a953200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:05:58.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:57 smithi145 bash[21623]: debug 2024-03-07T03:05:57.950+0000 7f288a953200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:05:58.088 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:57 smithi189 bash[24563]: debug 2024-03-07T03:05:57.808+0000 7f4f95d30200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:05:58.088 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:57 smithi189 bash[24563]: debug 2024-03-07T03:05:57.924+0000 7f4f95d30200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:05:58.088 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:58 smithi189 bash[24563]: debug 2024-03-07T03:05:58.028+0000 7f4f95d30200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:05:58.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:58 smithi145 bash[21623]: debug 2024-03-07T03:05:58.302+0000 7f288a953200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:05:58.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:58 smithi145 bash[21623]: debug 2024-03-07T03:05:58.546+0000 7f288a953200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:05:58.838 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:58 smithi189 bash[24563]: debug 2024-03-07T03:05:58.380+0000 7f4f95d30200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:05:58.838 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:58 smithi189 bash[24563]: debug 2024-03-07T03:05:58.628+0000 7f4f95d30200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:05:59.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:58 smithi145 bash[21623]: debug 2024-03-07T03:05:58.806+0000 7f288a953200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:05:59.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:58 smithi145 bash[21623]: debug 2024-03-07T03:05:58.918+0000 7f288a953200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:05:59.281 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:58 smithi189 bash[24563]: debug 2024-03-07T03:05:58.900+0000 7f4f95d30200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:05:59.281 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:59 smithi189 bash[24563]: debug 2024-03-07T03:05:59.012+0000 7f4f95d30200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:05:59.281 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:59 smithi189 bash[24563]: debug 2024-03-07T03:05:59.284+0000 7f4f95d30200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:05:59.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:59 smithi145 bash[21623]: debug 2024-03-07T03:05:59.190+0000 7f288a953200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:05:59.568 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:59 smithi145 bash[21623]: debug 2024-03-07T03:05:59.430+0000 7f288a953200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:05:59.588 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:59 smithi189 bash[24563]: debug 2024-03-07T03:05:59.524+0000 7f4f95d30200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:06:00.207 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:59 smithi145 bash[21623]: debug 2024-03-07T03:05:59.842+0000 7f288a953200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:06:00.208 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:05:59 smithi145 bash[21623]: debug 2024-03-07T03:05:59.958+0000 7f288a953200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:06:00.287 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:05:59 smithi189 bash[24563]: debug 2024-03-07T03:05:59.928+0000 7f4f95d30200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:06:00.287 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:00 smithi189 bash[24563]: debug 2024-03-07T03:06:00.044+0000 7f4f95d30200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:06:00.287 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:00 smithi189 bash[24563]: debug 2024-03-07T03:06:00.288+0000 7f4f95d30200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:06:00.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:00 smithi145 bash[21623]: debug 2024-03-07T03:06:00.206+0000 7f288a953200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:06:01.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:01 smithi145 bash[21623]: debug 2024-03-07T03:06:01.206+0000 7f288a953200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:06:01.567 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:01 smithi145 bash[21623]: debug 2024-03-07T03:06:01.322+0000 7f288a953200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:06:01.588 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:01 smithi189 bash[24563]: debug 2024-03-07T03:06:01.292+0000 7f4f95d30200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:06:01.588 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:01 smithi189 bash[24563]: debug 2024-03-07T03:06:01.408+0000 7f4f95d30200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:06:02.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:01 smithi145 bash[21623]: debug 2024-03-07T03:06:01.614+0000 7f288a953200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:06:02.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:01 smithi145 bash[21623]: debug 2024-03-07T03:06:01.730+0000 7f288a953200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:06:02.088 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:01 smithi189 bash[24563]: debug 2024-03-07T03:06:01.696+0000 7f4f95d30200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:06:02.088 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:01 smithi189 bash[24563]: debug 2024-03-07T03:06:01.812+0000 7f4f95d30200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:06:02.440 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21623]: debug 2024-03-07T03:06:02.178+0000 7f288a953200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:06:02.440 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21623]: debug 2024-03-07T03:06:02.302+0000 7f288a953200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: cluster 2024-03-07T03:06:02.330977+0000 mon.a (mon.0) 652 : cluster 1 Active manager daemon y restarted 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: cluster 2024-03-07T03:06:02.331538+0000 mon.a (mon.0) 653 : cluster 1 Activating manager daemon y 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: cluster 2024-03-07T03:06:02.348383+0000 mon.a (mon.0) 654 : cluster 0 osdmap e60: 8 total, 8 up, 8 in 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: cluster 2024-03-07T03:06:02.349502+0000 mon.a (mon.0) 655 : cluster 0 mgrmap e17: y(active, starting, since 0.0181496s), standbys: x 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.352587+0000 mon.c (mon.2) 12 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.352974+0000 mon.c (mon.2) 13 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.353437+0000 mon.c (mon.2) 14 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.354442+0000 mon.c (mon.2) 15 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.355142+0000 mon.c (mon.2) 16 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.355842+0000 mon.c (mon.2) 17 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.356483+0000 mon.c (mon.2) 18 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.357233+0000 mon.c (mon.2) 19 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.357842+0000 mon.c (mon.2) 20 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:06:02.441 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.358592+0000 mon.c (mon.2) 21 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:06:02.442 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.359214+0000 mon.c (mon.2) 22 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:02.442 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.359940+0000 mon.c (mon.2) 23 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:06:02.442 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.360644+0000 mon.c (mon.2) 24 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:06:02.442 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.361413+0000 mon.c (mon.2) 25 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:06:02.442 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.361808+0000 mon.c (mon.2) 26 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:06:02.442 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21339]: audit 2024-03-07T03:06:02.362972+0000 mon.c (mon.2) 27 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:06:02.442 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: cluster 2024-03-07T03:06:02.330977+0000 mon.a (mon.0) 652 : cluster 1 Active manager daemon y restarted 2024-03-07T03:06:02.442 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: cluster 2024-03-07T03:06:02.331538+0000 mon.a (mon.0) 653 : cluster 1 Activating manager daemon y 2024-03-07T03:06:02.442 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: cluster 2024-03-07T03:06:02.348383+0000 mon.a (mon.0) 654 : cluster 0 osdmap e60: 8 total, 8 up, 8 in 2024-03-07T03:06:02.442 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: cluster 2024-03-07T03:06:02.349502+0000 mon.a (mon.0) 655 : cluster 0 mgrmap e17: y(active, starting, since 0.0181496s), standbys: x 2024-03-07T03:06:02.442 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.352587+0000 mon.c (mon.2) 12 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:06:02.442 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.352974+0000 mon.c (mon.2) 13 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:06:02.442 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.353437+0000 mon.c (mon.2) 14 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:06:02.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.354442+0000 mon.c (mon.2) 15 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:06:02.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.355142+0000 mon.c (mon.2) 16 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:06:02.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.355842+0000 mon.c (mon.2) 17 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:06:02.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.356483+0000 mon.c (mon.2) 18 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:06:02.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.357233+0000 mon.c (mon.2) 19 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:06:02.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.357842+0000 mon.c (mon.2) 20 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:06:02.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.358592+0000 mon.c (mon.2) 21 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:06:02.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.359214+0000 mon.c (mon.2) 22 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:02.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.359940+0000 mon.c (mon.2) 23 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:06:02.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.360644+0000 mon.c (mon.2) 24 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:06:02.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.361413+0000 mon.c (mon.2) 25 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:06:02.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.361808+0000 mon.c (mon.2) 26 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:06:02.443 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[30906]: audit 2024-03-07T03:06:02.362972+0000 mon.c (mon.2) 27 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:06:02.501 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[24563]: debug 2024-03-07T03:06:02.252+0000 7f4f95d30200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:06:02.501 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[24563]: debug 2024-03-07T03:06:02.376+0000 7f4f95d30200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:06:02.502 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[24563]: [07/Mar/2024:03:06:02] ENGINE Bus STARTING 2024-03-07T03:06:02.502 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[24563]: CherryPy Checker: 2024-03-07T03:06:02.502 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[24563]: The Application mounted at '' has an empty config. 2024-03-07T03:06:02.502 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[24563]: [07/Mar/2024:03:06:02] ENGINE Serving on http://:::9283 2024-03-07T03:06:02.502 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[24563]: [07/Mar/2024:03:06:02] ENGINE Bus STARTED 2024-03-07T03:06:02.502 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: cluster 2024-03-07T03:06:02.330977+0000 mon.a (mon.0) 652 : cluster 1 Active manager daemon y restarted 2024-03-07T03:06:02.502 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: cluster 2024-03-07T03:06:02.331538+0000 mon.a (mon.0) 653 : cluster 1 Activating manager daemon y 2024-03-07T03:06:02.502 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: cluster 2024-03-07T03:06:02.348383+0000 mon.a (mon.0) 654 : cluster 0 osdmap e60: 8 total, 8 up, 8 in 2024-03-07T03:06:02.502 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: cluster 2024-03-07T03:06:02.349502+0000 mon.a (mon.0) 655 : cluster 0 mgrmap e17: y(active, starting, since 0.0181496s), standbys: x 2024-03-07T03:06:02.502 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.352587+0000 mon.c (mon.2) 12 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:06:02.502 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.352974+0000 mon.c (mon.2) 13 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:06:02.502 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.353437+0000 mon.c (mon.2) 14 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:06:02.503 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.354442+0000 mon.c (mon.2) 15 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:06:02.503 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.355142+0000 mon.c (mon.2) 16 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:06:02.503 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.355842+0000 mon.c (mon.2) 17 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:06:02.503 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.356483+0000 mon.c (mon.2) 18 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:06:02.503 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.357233+0000 mon.c (mon.2) 19 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:06:02.503 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.357842+0000 mon.c (mon.2) 20 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:06:02.503 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.358592+0000 mon.c (mon.2) 21 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:06:02.503 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.359214+0000 mon.c (mon.2) 22 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:02.503 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.359940+0000 mon.c (mon.2) 23 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:06:02.503 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.360644+0000 mon.c (mon.2) 24 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:06:02.503 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.361413+0000 mon.c (mon.2) 25 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:06:02.503 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.361808+0000 mon.c (mon.2) 26 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:06:02.503 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:02 smithi189 bash[23614]: audit 2024-03-07T03:06:02.362972+0000 mon.c (mon.2) 27 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:06:02.817 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21623]: [07/Mar/2024:03:06:02] ENGINE Bus STARTING 2024-03-07T03:06:02.817 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21623]: CherryPy Checker: 2024-03-07T03:06:02.817 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21623]: The Application mounted at '' has an empty config. 2024-03-07T03:06:02.817 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21623]: [07/Mar/2024:03:06:02] ENGINE Serving on http://:::9283 2024-03-07T03:06:02.817 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:02 smithi145 bash[21623]: [07/Mar/2024:03:06:02] ENGINE Bus STARTED 2024-03-07T03:06:03.437 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled alertmanager update... 2024-03-07T03:06:03.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: cluster 2024-03-07T03:06:02.393034+0000 mon.a (mon.0) 656 : cluster 1 Manager daemon y is now available 2024-03-07T03:06:03.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: cluster 2024-03-07T03:06:02.399936+0000 mon.a (mon.0) 657 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:06:03.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: cluster 2024-03-07T03:06:02.400073+0000 mon.a (mon.0) 658 : cluster 0 Standby manager daemon x started 2024-03-07T03:06:03.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: audit 2024-03-07T03:06:02.403722+0000 mon.b (mon.1) 26 : audit 0 from='mgr.? 172.21.15.189:0/3603196689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:06:03.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: audit 2024-03-07T03:06:02.406477+0000 mon.b (mon.1) 27 : audit 0 from='mgr.? 172.21.15.189:0/3603196689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:06:03.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: audit 2024-03-07T03:06:02.407617+0000 mon.b (mon.1) 28 : audit 0 from='mgr.? 172.21.15.189:0/3603196689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:06:03.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: audit 2024-03-07T03:06:02.407982+0000 mon.b (mon.1) 29 : audit 0 from='mgr.? 172.21.15.189:0/3603196689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:06:03.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: audit 2024-03-07T03:06:02.414136+0000 mon.c (mon.2) 28 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:03.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: audit 2024-03-07T03:06:02.416999+0000 mon.c (mon.2) 29 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:03.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: audit 2024-03-07T03:06:02.433880+0000 mon.a (mon.0) 659 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:03.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: audit 2024-03-07T03:06:02.438665+0000 mon.c (mon.2) 30 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:06:03.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: audit 2024-03-07T03:06:02.447416+0000 mon.c (mon.2) 31 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:06:03.674 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: audit 2024-03-07T03:06:02.447802+0000 mon.a (mon.0) 660 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:06:03.674 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: audit 2024-03-07T03:06:02.465268+0000 mon.c (mon.2) 32 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:06:03.674 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: audit 2024-03-07T03:06:02.465632+0000 mon.a (mon.0) 661 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:06:03.674 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: cephadm 2024-03-07T03:06:02.963668+0000 mgr.y (mgr.24302) 1 : cephadm 1 [07/Mar/2024:03:06:02] ENGINE Bus STARTING 2024-03-07T03:06:03.674 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: cephadm 2024-03-07T03:06:03.065627+0000 mgr.y (mgr.24302) 2 : cephadm 1 [07/Mar/2024:03:06:03] ENGINE Serving on http://172.21.15.145:8765 2024-03-07T03:06:03.674 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: cephadm 2024-03-07T03:06:03.179856+0000 mgr.y (mgr.24302) 3 : cephadm 1 [07/Mar/2024:03:06:03] ENGINE Serving on https://172.21.15.145:7150 2024-03-07T03:06:03.674 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:03 smithi189 bash[23614]: cephadm 2024-03-07T03:06:03.180059+0000 mgr.y (mgr.24302) 4 : cephadm 1 [07/Mar/2024:03:06:03] ENGINE Bus STARTED 2024-03-07T03:06:03.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: cluster 2024-03-07T03:06:02.393034+0000 mon.a (mon.0) 656 : cluster 1 Manager daemon y is now available 2024-03-07T03:06:03.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: cluster 2024-03-07T03:06:02.399936+0000 mon.a (mon.0) 657 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:06:03.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: cluster 2024-03-07T03:06:02.400073+0000 mon.a (mon.0) 658 : cluster 0 Standby manager daemon x started 2024-03-07T03:06:03.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: audit 2024-03-07T03:06:02.403722+0000 mon.b (mon.1) 26 : audit 0 from='mgr.? 172.21.15.189:0/3603196689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:06:03.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: audit 2024-03-07T03:06:02.406477+0000 mon.b (mon.1) 27 : audit 0 from='mgr.? 172.21.15.189:0/3603196689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:06:03.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: audit 2024-03-07T03:06:02.407617+0000 mon.b (mon.1) 28 : audit 0 from='mgr.? 172.21.15.189:0/3603196689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: audit 2024-03-07T03:06:02.407982+0000 mon.b (mon.1) 29 : audit 0 from='mgr.? 172.21.15.189:0/3603196689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: audit 2024-03-07T03:06:02.414136+0000 mon.c (mon.2) 28 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: audit 2024-03-07T03:06:02.416999+0000 mon.c (mon.2) 29 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: audit 2024-03-07T03:06:02.433880+0000 mon.a (mon.0) 659 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: audit 2024-03-07T03:06:02.438665+0000 mon.c (mon.2) 30 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: audit 2024-03-07T03:06:02.447416+0000 mon.c (mon.2) 31 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: audit 2024-03-07T03:06:02.447802+0000 mon.a (mon.0) 660 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: audit 2024-03-07T03:06:02.465268+0000 mon.c (mon.2) 32 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: audit 2024-03-07T03:06:02.465632+0000 mon.a (mon.0) 661 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: cephadm 2024-03-07T03:06:02.963668+0000 mgr.y (mgr.24302) 1 : cephadm 1 [07/Mar/2024:03:06:02] ENGINE Bus STARTING 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: cephadm 2024-03-07T03:06:03.065627+0000 mgr.y (mgr.24302) 2 : cephadm 1 [07/Mar/2024:03:06:03] ENGINE Serving on http://172.21.15.145:8765 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: cephadm 2024-03-07T03:06:03.179856+0000 mgr.y (mgr.24302) 3 : cephadm 1 [07/Mar/2024:03:06:03] ENGINE Serving on https://172.21.15.145:7150 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[21339]: cephadm 2024-03-07T03:06:03.180059+0000 mgr.y (mgr.24302) 4 : cephadm 1 [07/Mar/2024:03:06:03] ENGINE Bus STARTED 2024-03-07T03:06:03.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: cluster 2024-03-07T03:06:02.393034+0000 mon.a (mon.0) 656 : cluster 1 Manager daemon y is now available 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: cluster 2024-03-07T03:06:02.399936+0000 mon.a (mon.0) 657 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: cluster 2024-03-07T03:06:02.400073+0000 mon.a (mon.0) 658 : cluster 0 Standby manager daemon x started 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: audit 2024-03-07T03:06:02.403722+0000 mon.b (mon.1) 26 : audit 0 from='mgr.? 172.21.15.189:0/3603196689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: audit 2024-03-07T03:06:02.406477+0000 mon.b (mon.1) 27 : audit 0 from='mgr.? 172.21.15.189:0/3603196689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: audit 2024-03-07T03:06:02.407617+0000 mon.b (mon.1) 28 : audit 0 from='mgr.? 172.21.15.189:0/3603196689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: audit 2024-03-07T03:06:02.407982+0000 mon.b (mon.1) 29 : audit 0 from='mgr.? 172.21.15.189:0/3603196689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: audit 2024-03-07T03:06:02.414136+0000 mon.c (mon.2) 28 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: audit 2024-03-07T03:06:02.416999+0000 mon.c (mon.2) 29 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: audit 2024-03-07T03:06:02.433880+0000 mon.a (mon.0) 659 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: audit 2024-03-07T03:06:02.438665+0000 mon.c (mon.2) 30 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: audit 2024-03-07T03:06:02.447416+0000 mon.c (mon.2) 31 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: audit 2024-03-07T03:06:02.447802+0000 mon.a (mon.0) 660 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: audit 2024-03-07T03:06:02.465268+0000 mon.c (mon.2) 32 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: audit 2024-03-07T03:06:02.465632+0000 mon.a (mon.0) 661 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:06:03.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: cephadm 2024-03-07T03:06:02.963668+0000 mgr.y (mgr.24302) 1 : cephadm 1 [07/Mar/2024:03:06:02] ENGINE Bus STARTING 2024-03-07T03:06:03.821 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: cephadm 2024-03-07T03:06:03.065627+0000 mgr.y (mgr.24302) 2 : cephadm 1 [07/Mar/2024:03:06:03] ENGINE Serving on http://172.21.15.145:8765 2024-03-07T03:06:03.821 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: cephadm 2024-03-07T03:06:03.179856+0000 mgr.y (mgr.24302) 3 : cephadm 1 [07/Mar/2024:03:06:03] ENGINE Serving on https://172.21.15.145:7150 2024-03-07T03:06:03.821 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:03 smithi145 bash[30906]: cephadm 2024-03-07T03:06:03.180059+0000 mgr.y (mgr.24302) 4 : cephadm 1 [07/Mar/2024:03:06:03] ENGINE Bus STARTED 2024-03-07T03:06:04.279 DEBUG:teuthology.orchestra.run.smithi145:alertmanager.a> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@alertmanager.a.service 2024-03-07T03:06:04.282 INFO:tasks.cephadm:Adding grafana.a on smithi189 2024-03-07T03:06:04.282 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph orch apply grafana '1;smithi189=a' 2024-03-07T03:06:04.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:04 smithi145 bash[21339]: cluster 2024-03-07T03:06:03.418585+0000 mon.a (mon.0) 662 : cluster 0 mgrmap e18: y(active, since 1.08723s), standbys: x 2024-03-07T03:06:04.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:04 smithi145 bash[21339]: audit 2024-03-07T03:06:03.427375+0000 mgr.y (mgr.24302) 5 : audit 0 from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi145=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:06:04.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:04 smithi145 bash[21339]: cluster 2024-03-07T03:06:03.428676+0000 mgr.y (mgr.24302) 6 : cluster 0 pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:04.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:04 smithi145 bash[21339]: cephadm 2024-03-07T03:06:03.431813+0000 mgr.y (mgr.24302) 7 : cephadm 1 Saving service alertmanager spec with placement smithi145=a;count:1 2024-03-07T03:06:04.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:04 smithi145 bash[21339]: audit 2024-03-07T03:06:03.437957+0000 mon.a (mon.0) 663 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:04.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:04 smithi145 bash[21339]: cluster 2024-03-07T03:06:04.355910+0000 mgr.y (mgr.24302) 8 : cluster 0 pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:04.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:04 smithi145 bash[30906]: cluster 2024-03-07T03:06:03.418585+0000 mon.a (mon.0) 662 : cluster 0 mgrmap e18: y(active, since 1.08723s), standbys: x 2024-03-07T03:06:04.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:04 smithi145 bash[30906]: audit 2024-03-07T03:06:03.427375+0000 mgr.y (mgr.24302) 5 : audit 0 from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi145=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:06:04.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:04 smithi145 bash[30906]: cluster 2024-03-07T03:06:03.428676+0000 mgr.y (mgr.24302) 6 : cluster 0 pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:04.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:04 smithi145 bash[30906]: cephadm 2024-03-07T03:06:03.431813+0000 mgr.y (mgr.24302) 7 : cephadm 1 Saving service alertmanager spec with placement smithi145=a;count:1 2024-03-07T03:06:04.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:04 smithi145 bash[30906]: audit 2024-03-07T03:06:03.437957+0000 mon.a (mon.0) 663 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:04.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:04 smithi145 bash[30906]: cluster 2024-03-07T03:06:04.355910+0000 mgr.y (mgr.24302) 8 : cluster 0 pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:04.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:04 smithi189 bash[23614]: cluster 2024-03-07T03:06:03.418585+0000 mon.a (mon.0) 662 : cluster 0 mgrmap e18: y(active, since 1.08723s), standbys: x 2024-03-07T03:06:04.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:04 smithi189 bash[23614]: audit 2024-03-07T03:06:03.427375+0000 mgr.y (mgr.24302) 5 : audit 0 from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi145=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:06:04.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:04 smithi189 bash[23614]: cluster 2024-03-07T03:06:03.428676+0000 mgr.y (mgr.24302) 6 : cluster 0 pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:04.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:04 smithi189 bash[23614]: cephadm 2024-03-07T03:06:03.431813+0000 mgr.y (mgr.24302) 7 : cephadm 1 Saving service alertmanager spec with placement smithi145=a;count:1 2024-03-07T03:06:04.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:04 smithi189 bash[23614]: audit 2024-03-07T03:06:03.437957+0000 mon.a (mon.0) 663 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:04.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:04 smithi189 bash[23614]: cluster 2024-03-07T03:06:04.355910+0000 mgr.y (mgr.24302) 8 : cluster 0 pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:05.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:05 smithi145 bash[21339]: cluster 2024-03-07T03:06:04.451387+0000 mon.a (mon.0) 664 : cluster 0 mgrmap e19: y(active, since 2s), standbys: x 2024-03-07T03:06:05.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:05 smithi145 bash[30906]: cluster 2024-03-07T03:06:04.451387+0000 mon.a (mon.0) 664 : cluster 0 mgrmap e19: y(active, since 2s), standbys: x 2024-03-07T03:06:05.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:05 smithi189 bash[23614]: cluster 2024-03-07T03:06:04.451387+0000 mon.a (mon.0) 664 : cluster 0 mgrmap e19: y(active, since 2s), standbys: x 2024-03-07T03:06:06.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:06 smithi145 bash[21339]: cluster 2024-03-07T03:06:06.356503+0000 mgr.y (mgr.24302) 9 : cluster 0 pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:06.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:06 smithi145 bash[30906]: cluster 2024-03-07T03:06:06.356503+0000 mgr.y (mgr.24302) 9 : cluster 0 pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:06.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:06 smithi189 bash[23614]: cluster 2024-03-07T03:06:06.356503+0000 mgr.y (mgr.24302) 9 : cluster 0 pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:07.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:07 smithi145 bash[21339]: cluster 2024-03-07T03:06:06.463928+0000 mon.a (mon.0) 665 : cluster 0 mgrmap e20: y(active, since 4s), standbys: x 2024-03-07T03:06:07.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:07 smithi145 bash[30906]: cluster 2024-03-07T03:06:06.463928+0000 mon.a (mon.0) 665 : cluster 0 mgrmap e20: y(active, since 4s), standbys: x 2024-03-07T03:06:07.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:07 smithi189 bash[23614]: cluster 2024-03-07T03:06:06.463928+0000 mon.a (mon.0) 665 : cluster 0 mgrmap e20: y(active, since 4s), standbys: x 2024-03-07T03:06:08.799 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:06:09.317 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[21339]: audit 2024-03-07T03:06:07.983959+0000 mon.a (mon.0) 666 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.317 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[21339]: audit 2024-03-07T03:06:07.992464+0000 mon.a (mon.0) 667 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.317 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[21339]: audit 2024-03-07T03:06:08.247691+0000 mon.a (mon.0) 668 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[21339]: audit 2024-03-07T03:06:08.263119+0000 mon.a (mon.0) 669 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[21339]: cluster 2024-03-07T03:06:08.356942+0000 mgr.y (mgr.24302) 10 : cluster 0 pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:09.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[21339]: audit 2024-03-07T03:06:08.796750+0000 mon.a (mon.0) 670 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[21339]: audit 2024-03-07T03:06:08.810602+0000 mon.a (mon.0) 671 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.318 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[30906]: audit 2024-03-07T03:06:07.983959+0000 mon.a (mon.0) 666 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.318 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[30906]: audit 2024-03-07T03:06:07.992464+0000 mon.a (mon.0) 667 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.318 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[30906]: audit 2024-03-07T03:06:08.247691+0000 mon.a (mon.0) 668 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.318 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[30906]: audit 2024-03-07T03:06:08.263119+0000 mon.a (mon.0) 669 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.318 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[30906]: cluster 2024-03-07T03:06:08.356942+0000 mgr.y (mgr.24302) 10 : cluster 0 pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:09.318 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[30906]: audit 2024-03-07T03:06:08.796750+0000 mon.a (mon.0) 670 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.318 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:08 smithi145 bash[30906]: audit 2024-03-07T03:06:08.810602+0000 mon.a (mon.0) 671 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:08 smithi189 bash[23614]: audit 2024-03-07T03:06:07.983959+0000 mon.a (mon.0) 666 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:08 smithi189 bash[23614]: audit 2024-03-07T03:06:07.992464+0000 mon.a (mon.0) 667 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:08 smithi189 bash[23614]: audit 2024-03-07T03:06:08.247691+0000 mon.a (mon.0) 668 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:08 smithi189 bash[23614]: audit 2024-03-07T03:06:08.263119+0000 mon.a (mon.0) 669 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:08 smithi189 bash[23614]: cluster 2024-03-07T03:06:08.356942+0000 mgr.y (mgr.24302) 10 : cluster 0 pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:09.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:08 smithi189 bash[23614]: audit 2024-03-07T03:06:08.796750+0000 mon.a (mon.0) 670 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:09.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:08 smithi189 bash[23614]: audit 2024-03-07T03:06:08.810602+0000 mon.a (mon.0) 671 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.318 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.064981+0000 mon.a (mon.0) 672 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.081463+0000 mon.a (mon.0) 673 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.085700+0000 mon.c (mon.2) 33 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.086506+0000 mon.a (mon.0) 674 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.088913+0000 mon.c (mon.2) 34 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.089732+0000 mon.a (mon.0) 675 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.091983+0000 mon.c (mon.2) 35 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.092769+0000 mon.a (mon.0) 676 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.095258+0000 mon.c (mon.2) 36 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.096148+0000 mon.a (mon.0) 677 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: cephadm 2024-03-07T03:06:09.097989+0000 mgr.y (mgr.24302) 11 : cephadm 1 Adjusting osd_memory_target on smithi189 to 4060M 2024-03-07T03:06:10.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.109640+0000 mon.a (mon.0) 678 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.114211+0000 mon.c (mon.2) 37 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:10.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.116110+0000 mon.c (mon.2) 38 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:10.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: cephadm 2024-03-07T03:06:09.118740+0000 mgr.y (mgr.24302) 12 : cephadm 1 Updating smithi145:/etc/ceph/ceph.conf 2024-03-07T03:06:10.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: cephadm 2024-03-07T03:06:09.119227+0000 mgr.y (mgr.24302) 13 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T03:06:10.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.890828+0000 mon.a (mon.0) 679 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.898471+0000 mon.a (mon.0) 680 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.910378+0000 mon.a (mon.0) 681 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.919320+0000 mon.a (mon.0) 682 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[21339]: audit 2024-03-07T03:06:09.930738+0000 mon.a (mon.0) 683 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.064981+0000 mon.a (mon.0) 672 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.081463+0000 mon.a (mon.0) 673 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.085700+0000 mon.c (mon.2) 33 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.086506+0000 mon.a (mon.0) 674 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.088913+0000 mon.c (mon.2) 34 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.089732+0000 mon.a (mon.0) 675 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.091983+0000 mon.c (mon.2) 35 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.092769+0000 mon.a (mon.0) 676 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.095258+0000 mon.c (mon.2) 36 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.096148+0000 mon.a (mon.0) 677 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: cephadm 2024-03-07T03:06:09.097989+0000 mgr.y (mgr.24302) 11 : cephadm 1 Adjusting osd_memory_target on smithi189 to 4060M 2024-03-07T03:06:10.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.109640+0000 mon.a (mon.0) 678 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.114211+0000 mon.c (mon.2) 37 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:10.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.116110+0000 mon.c (mon.2) 38 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:10.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: cephadm 2024-03-07T03:06:09.118740+0000 mgr.y (mgr.24302) 12 : cephadm 1 Updating smithi145:/etc/ceph/ceph.conf 2024-03-07T03:06:10.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: cephadm 2024-03-07T03:06:09.119227+0000 mgr.y (mgr.24302) 13 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T03:06:10.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.890828+0000 mon.a (mon.0) 679 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.898471+0000 mon.a (mon.0) 680 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.910378+0000 mon.a (mon.0) 681 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.919320+0000 mon.a (mon.0) 682 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[30906]: audit 2024-03-07T03:06:09.930738+0000 mon.a (mon.0) 683 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.064981+0000 mon.a (mon.0) 672 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.081463+0000 mon.a (mon.0) 673 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.085700+0000 mon.c (mon.2) 33 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.086506+0000 mon.a (mon.0) 674 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.088913+0000 mon.c (mon.2) 34 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.089732+0000 mon.a (mon.0) 675 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.091983+0000 mon.c (mon.2) 35 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.092769+0000 mon.a (mon.0) 676 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.095258+0000 mon.c (mon.2) 36 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.096148+0000 mon.a (mon.0) 677 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: cephadm 2024-03-07T03:06:09.097989+0000 mgr.y (mgr.24302) 11 : cephadm 1 Adjusting osd_memory_target on smithi189 to 4060M 2024-03-07T03:06:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.109640+0000 mon.a (mon.0) 678 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.114211+0000 mon.c (mon.2) 37 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.116110+0000 mon.c (mon.2) 38 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: cephadm 2024-03-07T03:06:09.118740+0000 mgr.y (mgr.24302) 12 : cephadm 1 Updating smithi145:/etc/ceph/ceph.conf 2024-03-07T03:06:10.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: cephadm 2024-03-07T03:06:09.119227+0000 mgr.y (mgr.24302) 13 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T03:06:10.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.890828+0000 mon.a (mon.0) 679 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.898471+0000 mon.a (mon.0) 680 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.910378+0000 mon.a (mon.0) 681 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.919320+0000 mon.a (mon.0) 682 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:10 smithi189 bash[23614]: audit 2024-03-07T03:06:09.930738+0000 mon.a (mon.0) 683 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:10.611 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.611 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.612 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.612 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.612 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.613 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.613 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.896 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.897 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.897 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.897 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.898 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.898 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.898 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.899 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.899 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.899 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:10.899 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: Started Ceph node-exporter.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:06:10.899 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:10 smithi145 bash[55620]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-03-07T03:06:10.899 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:06:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[21339]: cephadm 2024-03-07T03:06:09.386399+0000 mgr.y (mgr.24302) 14 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:06:11.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[21339]: cephadm 2024-03-07T03:06:09.386757+0000 mgr.y (mgr.24302) 15 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:06:11.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[21339]: cephadm 2024-03-07T03:06:09.551569+0000 mgr.y (mgr.24302) 16 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:06:11.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[21339]: cephadm 2024-03-07T03:06:09.551766+0000 mgr.y (mgr.24302) 17 : cephadm 1 Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:06:11.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[21339]: cephadm 2024-03-07T03:06:09.716212+0000 mgr.y (mgr.24302) 18 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:06:11.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[21339]: cephadm 2024-03-07T03:06:09.716538+0000 mgr.y (mgr.24302) 19 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:06:11.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[21339]: cephadm 2024-03-07T03:06:09.934931+0000 mgr.y (mgr.24302) 20 : cephadm 1 Deploying daemon node-exporter.a on smithi145 2024-03-07T03:06:11.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[21339]: cluster 2024-03-07T03:06:10.357577+0000 mgr.y (mgr.24302) 21 : cluster 0 pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-07T03:06:11.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[21339]: audit 2024-03-07T03:06:10.893433+0000 mon.a (mon.0) 684 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:11.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[21339]: audit 2024-03-07T03:06:10.902988+0000 mon.a (mon.0) 685 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:11.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[21339]: audit 2024-03-07T03:06:10.958371+0000 mon.a (mon.0) 686 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:11.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[30906]: cephadm 2024-03-07T03:06:09.386399+0000 mgr.y (mgr.24302) 14 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:06:11.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[30906]: cephadm 2024-03-07T03:06:09.386757+0000 mgr.y (mgr.24302) 15 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:06:11.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[30906]: cephadm 2024-03-07T03:06:09.551569+0000 mgr.y (mgr.24302) 16 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:06:11.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[30906]: cephadm 2024-03-07T03:06:09.551766+0000 mgr.y (mgr.24302) 17 : cephadm 1 Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:06:11.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[30906]: cephadm 2024-03-07T03:06:09.716212+0000 mgr.y (mgr.24302) 18 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:06:11.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[30906]: cephadm 2024-03-07T03:06:09.716538+0000 mgr.y (mgr.24302) 19 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:06:11.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[30906]: cephadm 2024-03-07T03:06:09.934931+0000 mgr.y (mgr.24302) 20 : cephadm 1 Deploying daemon node-exporter.a on smithi145 2024-03-07T03:06:11.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[30906]: cluster 2024-03-07T03:06:10.357577+0000 mgr.y (mgr.24302) 21 : cluster 0 pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-07T03:06:11.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[30906]: audit 2024-03-07T03:06:10.893433+0000 mon.a (mon.0) 684 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:11.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[30906]: audit 2024-03-07T03:06:10.902988+0000 mon.a (mon.0) 685 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:11.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[30906]: audit 2024-03-07T03:06:10.958371+0000 mon.a (mon.0) 686 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:11.337 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:11 smithi189 bash[23614]: cephadm 2024-03-07T03:06:09.386399+0000 mgr.y (mgr.24302) 14 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:06:11.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:11 smithi189 bash[23614]: cephadm 2024-03-07T03:06:09.386757+0000 mgr.y (mgr.24302) 15 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:06:11.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:11 smithi189 bash[23614]: cephadm 2024-03-07T03:06:09.551569+0000 mgr.y (mgr.24302) 16 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:06:11.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:11 smithi189 bash[23614]: cephadm 2024-03-07T03:06:09.551766+0000 mgr.y (mgr.24302) 17 : cephadm 1 Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:06:11.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:11 smithi189 bash[23614]: cephadm 2024-03-07T03:06:09.716212+0000 mgr.y (mgr.24302) 18 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:06:11.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:11 smithi189 bash[23614]: cephadm 2024-03-07T03:06:09.716538+0000 mgr.y (mgr.24302) 19 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:06:11.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:11 smithi189 bash[23614]: cephadm 2024-03-07T03:06:09.934931+0000 mgr.y (mgr.24302) 20 : cephadm 1 Deploying daemon node-exporter.a on smithi145 2024-03-07T03:06:11.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:11 smithi189 bash[23614]: cluster 2024-03-07T03:06:10.357577+0000 mgr.y (mgr.24302) 21 : cluster 0 pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-07T03:06:11.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:11 smithi189 bash[23614]: audit 2024-03-07T03:06:10.893433+0000 mon.a (mon.0) 684 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:11.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:11 smithi189 bash[23614]: audit 2024-03-07T03:06:10.902988+0000 mon.a (mon.0) 685 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:11.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:11 smithi189 bash[23614]: audit 2024-03-07T03:06:10.958371+0000 mon.a (mon.0) 686 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:11.575 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled grafana update... 2024-03-07T03:06:11.592 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.592 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.593 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.593 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.593 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.817 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[55620]: v1.5.0: Pulling from prometheus/node-exporter 2024-03-07T03:06:11.862 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.862 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.862 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.863 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.863 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.863 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.863 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.863 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.863 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.863 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.863 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: Started Ceph node-exporter.b for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:06:11.864 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:11.864 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:06:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:12.074 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[55620]: 22b70bddd3ac: Pulling fs layer 2024-03-07T03:06:12.074 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[55620]: 5c12815fee55: Pulling fs layer 2024-03-07T03:06:12.074 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:11 smithi145 bash[55620]: c0e87333d380: Pulling fs layer 2024-03-07T03:06:12.074 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[55620]: 5c12815fee55: Verifying Checksum 2024-03-07T03:06:12.075 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[55620]: 5c12815fee55: Download complete 2024-03-07T03:06:12.075 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[55620]: 22b70bddd3ac: Verifying Checksum 2024-03-07T03:06:12.075 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[55620]: 22b70bddd3ac: Download complete 2024-03-07T03:06:12.338 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:11 smithi189 bash[48779]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-03-07T03:06:12.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:12 smithi189 bash[23614]: cephadm 2024-03-07T03:06:10.959618+0000 mgr.y (mgr.24302) 22 : cephadm 1 Deploying daemon node-exporter.b on smithi189 2024-03-07T03:06:12.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:12 smithi189 bash[23614]: audit 2024-03-07T03:06:11.576072+0000 mon.a (mon.0) 687 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:12 smithi189 bash[23614]: audit 2024-03-07T03:06:11.878097+0000 mon.a (mon.0) 688 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:12 smithi189 bash[23614]: audit 2024-03-07T03:06:11.892641+0000 mon.a (mon.0) 689 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:12 smithi189 bash[23614]: audit 2024-03-07T03:06:11.936431+0000 mon.a (mon.0) 690 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:12 smithi189 bash[23614]: audit 2024-03-07T03:06:11.946885+0000 mon.a (mon.0) 691 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:12 smithi189 bash[23614]: audit 2024-03-07T03:06:11.963746+0000 mon.a (mon.0) 692 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[30906]: cephadm 2024-03-07T03:06:10.959618+0000 mgr.y (mgr.24302) 22 : cephadm 1 Deploying daemon node-exporter.b on smithi189 2024-03-07T03:06:12.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[30906]: audit 2024-03-07T03:06:11.576072+0000 mon.a (mon.0) 687 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[30906]: audit 2024-03-07T03:06:11.878097+0000 mon.a (mon.0) 688 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[30906]: audit 2024-03-07T03:06:11.892641+0000 mon.a (mon.0) 689 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.379 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[30906]: audit 2024-03-07T03:06:11.936431+0000 mon.a (mon.0) 690 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[30906]: audit 2024-03-07T03:06:11.946885+0000 mon.a (mon.0) 691 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.380 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[30906]: audit 2024-03-07T03:06:11.963746+0000 mon.a (mon.0) 692 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[21339]: cephadm 2024-03-07T03:06:10.959618+0000 mgr.y (mgr.24302) 22 : cephadm 1 Deploying daemon node-exporter.b on smithi189 2024-03-07T03:06:12.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[21339]: audit 2024-03-07T03:06:11.576072+0000 mon.a (mon.0) 687 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[21339]: audit 2024-03-07T03:06:11.878097+0000 mon.a (mon.0) 688 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[21339]: audit 2024-03-07T03:06:11.892641+0000 mon.a (mon.0) 689 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[21339]: audit 2024-03-07T03:06:11.936431+0000 mon.a (mon.0) 690 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[21339]: audit 2024-03-07T03:06:11.946885+0000 mon.a (mon.0) 691 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.380 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[21339]: audit 2024-03-07T03:06:11.963746+0000 mon.a (mon.0) 692 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:12.381 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:12 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:12.381 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[55620]: c0e87333d380: Verifying Checksum 2024-03-07T03:06:12.381 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[55620]: c0e87333d380: Download complete 2024-03-07T03:06:12.381 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[55620]: 22b70bddd3ac: Pull complete 2024-03-07T03:06:12.660 DEBUG:teuthology.orchestra.run.smithi189:grafana.a> sudo journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@grafana.a.service 2024-03-07T03:06:12.662 INFO:tasks.cephadm:Setting up client nodes... 2024-03-07T03:06:12.663 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-07T03:06:12.673 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:06:12] "GET /metrics HTTP/1.1" 200 23683 "" "Prometheus/2.43.0" 2024-03-07T03:06:12.674 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:12 smithi145 bash[55620]: 5c12815fee55: Pull complete 2024-03-07T03:06:12.967 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:12 smithi189 bash[48779]: v1.5.0: Pulling from prometheus/node-exporter 2024-03-07T03:06:13.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:13 smithi189 bash[23614]: audit 2024-03-07T03:06:11.560565+0000 mgr.y (mgr.24302) 23 : audit 0 from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:06:13.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:13 smithi189 bash[23614]: cephadm 2024-03-07T03:06:11.564899+0000 mgr.y (mgr.24302) 24 : cephadm 1 Saving service grafana spec with placement smithi189=a;count:1 2024-03-07T03:06:13.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:13 smithi189 bash[23614]: cephadm 2024-03-07T03:06:11.989247+0000 mgr.y (mgr.24302) 25 : cephadm 1 Deploying daemon alertmanager.a on smithi145 2024-03-07T03:06:13.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:13 smithi189 bash[23614]: cluster 2024-03-07T03:06:12.357981+0000 mgr.y (mgr.24302) 26 : cluster 0 pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-07T03:06:13.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:13 smithi189 bash[23614]: audit 2024-03-07T03:06:12.438831+0000 mon.a (mon.0) 693 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:13.338 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:12 smithi189 bash[48779]: 22b70bddd3ac: Pulling fs layer 2024-03-07T03:06:13.338 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:12 smithi189 bash[48779]: 5c12815fee55: Pulling fs layer 2024-03-07T03:06:13.338 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:12 smithi189 bash[48779]: c0e87333d380: Pulling fs layer 2024-03-07T03:06:13.339 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:13 smithi189 bash[48779]: 5c12815fee55: Verifying Checksum 2024-03-07T03:06:13.339 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:13 smithi189 bash[48779]: 5c12815fee55: Download complete 2024-03-07T03:06:13.339 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:13 smithi189 bash[48779]: 22b70bddd3ac: Verifying Checksum 2024-03-07T03:06:13.339 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:13 smithi189 bash[48779]: 22b70bddd3ac: Download complete 2024-03-07T03:06:13.352 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:13 smithi145 bash[55620]: c0e87333d380: Pull complete 2024-03-07T03:06:13.352 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:13 smithi145 bash[30906]: audit 2024-03-07T03:06:11.560565+0000 mgr.y (mgr.24302) 23 : audit 0 from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:06:13.352 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:13 smithi145 bash[30906]: cephadm 2024-03-07T03:06:11.564899+0000 mgr.y (mgr.24302) 24 : cephadm 1 Saving service grafana spec with placement smithi189=a;count:1 2024-03-07T03:06:13.352 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:13 smithi145 bash[30906]: cephadm 2024-03-07T03:06:11.989247+0000 mgr.y (mgr.24302) 25 : cephadm 1 Deploying daemon alertmanager.a on smithi145 2024-03-07T03:06:13.352 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:13 smithi145 bash[30906]: cluster 2024-03-07T03:06:12.357981+0000 mgr.y (mgr.24302) 26 : cluster 0 pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-07T03:06:13.352 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:13 smithi145 bash[30906]: audit 2024-03-07T03:06:12.438831+0000 mon.a (mon.0) 693 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:13.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:13 smithi145 bash[21339]: audit 2024-03-07T03:06:11.560565+0000 mgr.y (mgr.24302) 23 : audit 0 from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:06:13.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:13 smithi145 bash[21339]: cephadm 2024-03-07T03:06:11.564899+0000 mgr.y (mgr.24302) 24 : cephadm 1 Saving service grafana spec with placement smithi189=a;count:1 2024-03-07T03:06:13.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:13 smithi145 bash[21339]: cephadm 2024-03-07T03:06:11.989247+0000 mgr.y (mgr.24302) 25 : cephadm 1 Deploying daemon alertmanager.a on smithi145 2024-03-07T03:06:13.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:13 smithi145 bash[21339]: cluster 2024-03-07T03:06:12.357981+0000 mgr.y (mgr.24302) 26 : cluster 0 pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-07T03:06:13.353 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:13 smithi145 bash[21339]: audit 2024-03-07T03:06:12.438831+0000 mon.a (mon.0) 693 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:13.793 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:13 smithi189 bash[48779]: 22b70bddd3ac: Pull complete 2024-03-07T03:06:13.794 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:13 smithi189 bash[48779]: c0e87333d380: Verifying Checksum 2024-03-07T03:06:13.794 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:13 smithi189 bash[48779]: c0e87333d380: Download complete 2024-03-07T03:06:13.817 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:13 smithi145 bash[55620]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-03-07T03:06:13.817 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:13 smithi145 bash[55620]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-03-07T03:06:14.415 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:13 smithi189 bash[48779]: 5c12815fee55: Pull complete 2024-03-07T03:06:14.415 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:14 smithi189 bash[48779]: c0e87333d380: Pull complete 2024-03-07T03:06:14.415 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:14 smithi189 bash[48779]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-03-07T03:06:14.415 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:14 smithi189 bash[48779]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-03-07T03:06:14.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:14 smithi145 bash[30906]: cluster 2024-03-07T03:06:14.358604+0000 mgr.y (mgr.24302) 27 : cluster 0 pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-07T03:06:14.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:14 smithi145 bash[21339]: cluster 2024-03-07T03:06:14.358604+0000 mgr.y (mgr.24302) 27 : cluster 0 pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-07T03:06:14.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:14 smithi189 bash[23614]: cluster 2024-03-07T03:06:14.358604+0000 mgr.y (mgr.24302) 27 : cluster 0 pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-07T03:06:15.838 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.523Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.523Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.524Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.524Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.524Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.524Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=arp 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-07T03:06:15.839 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-07T03:06:15.840 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-07T03:06:15.840 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=edac 2024-03-07T03:06:15.840 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-07T03:06:15.840 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-07T03:06:15.840 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-07T03:06:15.840 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-07T03:06:15.840 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-07T03:06:15.840 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-07T03:06:15.840 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-07T03:06:15.840 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-07T03:06:15.840 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=os 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-07T03:06:15.841 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=stat 2024-03-07T03:06:15.842 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-07T03:06:15.842 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-07T03:06:15.842 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-07T03:06:15.842 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=time 2024-03-07T03:06:15.842 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-07T03:06:15.842 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=uname 2024-03-07T03:06:15.842 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-07T03:06:15.842 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-07T03:06:15.842 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-07T03:06:15.842 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-07T03:06:15.842 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:15 smithi189 bash[48779]: ts=2024-03-07T03:06:15.525Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-07T03:06:16.317 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.017Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-07T03:06:16.318 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.017Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-07T03:06:16.318 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-07T03:06:16.318 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-07T03:06:16.318 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-07T03:06:16.318 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-07T03:06:16.318 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-07T03:06:16.318 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=arp 2024-03-07T03:06:16.318 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-07T03:06:16.318 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-07T03:06:16.318 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-07T03:06:16.318 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-07T03:06:16.318 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=edac 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.019Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-07T03:06:16.319 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=os 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-07T03:06:16.320 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=stat 2024-03-07T03:06:16.321 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-07T03:06:16.321 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-07T03:06:16.321 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-07T03:06:16.321 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=time 2024-03-07T03:06:16.321 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-07T03:06:16.321 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=uname 2024-03-07T03:06:16.321 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-07T03:06:16.321 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-07T03:06:16.321 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-07T03:06:16.321 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-07T03:06:16.321 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[55620]: ts=2024-03-07T03:06:16.020Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-07T03:06:16.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[21339]: cluster 2024-03-07T03:06:16.359031+0000 mgr.y (mgr.24302) 28 : cluster 0 pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:06:16.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:16 smithi145 bash[30906]: cluster 2024-03-07T03:06:16.359031+0000 mgr.y (mgr.24302) 28 : cluster 0 pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:06:16.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:16 smithi189 bash[23614]: cluster 2024-03-07T03:06:16.359031+0000 mgr.y (mgr.24302) 28 : cluster 0 pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:06:17.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:17 smithi145 bash[21339]: audit 2024-03-07T03:06:17.439984+0000 mon.c (mon.2) 39 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:06:17.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:17 smithi145 bash[30906]: audit 2024-03-07T03:06:17.439984+0000 mon.c (mon.2) 39 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:06:17.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:17 smithi189 bash[23614]: audit 2024-03-07T03:06:17.439984+0000 mon.c (mon.2) 39 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:06:18.397 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:06:19.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:19 smithi145 bash[21339]: cluster 2024-03-07T03:06:18.359591+0000 mgr.y (mgr.24302) 29 : cluster 0 pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:06:19.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:19 smithi145 bash[30906]: cluster 2024-03-07T03:06:18.359591+0000 mgr.y (mgr.24302) 29 : cluster 0 pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:06:19.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:19 smithi189 bash[23614]: cluster 2024-03-07T03:06:18.359591+0000 mgr.y (mgr.24302) 29 : cluster 0 pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:06:20.721 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:20 smithi145 bash[21339]: cluster 2024-03-07T03:06:20.360281+0000 mgr.y (mgr.24302) 30 : cluster 0 pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:06:20.721 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:20 smithi145 bash[30906]: cluster 2024-03-07T03:06:20.360281+0000 mgr.y (mgr.24302) 30 : cluster 0 pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:06:20.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:20 smithi189 bash[23614]: cluster 2024-03-07T03:06:20.360281+0000 mgr.y (mgr.24302) 30 : cluster 0 pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:06:22.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:22 smithi145 bash[21339]: cluster 2024-03-07T03:06:22.360897+0000 mgr.y (mgr.24302) 31 : cluster 0 pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:22.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:22 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:06:22] "GET /metrics HTTP/1.1" 200 26212 "" "Prometheus/2.43.0" 2024-03-07T03:06:22.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:22 smithi145 bash[30906]: cluster 2024-03-07T03:06:22.360897+0000 mgr.y (mgr.24302) 31 : cluster 0 pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:22.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:22 smithi189 bash[23614]: cluster 2024-03-07T03:06:22.360897+0000 mgr.y (mgr.24302) 31 : cluster 0 pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:23.375 INFO:teuthology.orchestra.run.smithi145.stdout:[client.0] 2024-03-07T03:06:23.375 INFO:teuthology.orchestra.run.smithi145.stdout: key = AQAvL+llOH0CFhAAdGl2uXclZUkOnnaL+ynhTw== 2024-03-07T03:06:24.471 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T03:06:24.471 DEBUG:teuthology.orchestra.run.smithi145:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-03-07T03:06:24.471 DEBUG:teuthology.orchestra.run.smithi145:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-03-07T03:06:24.535 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-07T03:06:24.557 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:24 smithi145 bash[30906]: audit 2024-03-07T03:06:23.368585+0000 mon.c (mon.2) 40 : audit 1 from='client.? 172.21.15.145:0/1697312986' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:06:24.557 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:24 smithi145 bash[30906]: audit 2024-03-07T03:06:23.369130+0000 mon.a (mon.0) 694 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:06:24.557 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:24 smithi145 bash[30906]: audit 2024-03-07T03:06:23.375526+0000 mon.a (mon.0) 695 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-07T03:06:24.557 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:24 smithi145 bash[21339]: audit 2024-03-07T03:06:23.368585+0000 mon.c (mon.2) 40 : audit 1 from='client.? 172.21.15.145:0/1697312986' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:06:24.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:24 smithi145 bash[21339]: audit 2024-03-07T03:06:23.369130+0000 mon.a (mon.0) 694 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:06:24.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:24 smithi145 bash[21339]: audit 2024-03-07T03:06:23.375526+0000 mon.a (mon.0) 695 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-07T03:06:24.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:24 smithi189 bash[23614]: audit 2024-03-07T03:06:23.368585+0000 mon.c (mon.2) 40 : audit 1 from='client.? 172.21.15.145:0/1697312986' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:06:24.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:24 smithi189 bash[23614]: audit 2024-03-07T03:06:23.369130+0000 mon.a (mon.0) 694 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:06:24.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:24 smithi189 bash[23614]: audit 2024-03-07T03:06:23.375526+0000 mon.a (mon.0) 695 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-07T03:06:24.803 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:24.803 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:24.803 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:24.804 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:24.804 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:24.804 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:24.804 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:24.804 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:24.805 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:24.805 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:24.805 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:24.805 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:24.807 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:24.807 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:25.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:25.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:25.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:25.068 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:25.068 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:25.068 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:25.068 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:24 smithi145 systemd[1]: Started Ceph alertmanager.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:06:25.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:25 smithi145 bash[21339]: cluster 2024-03-07T03:06:24.361579+0000 mgr.y (mgr.24302) 32 : cluster 0 pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:25.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:25 smithi145 bash[21339]: audit 2024-03-07T03:06:24.958139+0000 mon.a (mon.0) 696 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:25.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:25 smithi145 bash[21339]: audit 2024-03-07T03:06:24.970396+0000 mon.a (mon.0) 697 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:25.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:25 smithi145 bash[21339]: audit 2024-03-07T03:06:24.990525+0000 mon.a (mon.0) 698 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:25.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:25 smithi145 bash[21339]: audit 2024-03-07T03:06:24.994728+0000 mon.a (mon.0) 699 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:25.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:25 smithi145 bash[21339]: audit 2024-03-07T03:06:25.043601+0000 mon.c (mon.2) 41 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:25.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:25 smithi145 bash[30906]: cluster 2024-03-07T03:06:24.361579+0000 mgr.y (mgr.24302) 32 : cluster 0 pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:25.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:25 smithi145 bash[30906]: audit 2024-03-07T03:06:24.958139+0000 mon.a (mon.0) 696 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:25.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:25 smithi145 bash[30906]: audit 2024-03-07T03:06:24.970396+0000 mon.a (mon.0) 697 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:25.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:25 smithi145 bash[30906]: audit 2024-03-07T03:06:24.990525+0000 mon.a (mon.0) 698 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:25.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:25 smithi145 bash[30906]: audit 2024-03-07T03:06:24.994728+0000 mon.a (mon.0) 699 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:25.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:25 smithi145 bash[30906]: audit 2024-03-07T03:06:25.043601+0000 mon.c (mon.2) 41 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:25.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:25 smithi189 bash[23614]: cluster 2024-03-07T03:06:24.361579+0000 mgr.y (mgr.24302) 32 : cluster 0 pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:25.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:25 smithi189 bash[23614]: audit 2024-03-07T03:06:24.958139+0000 mon.a (mon.0) 696 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:25.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:25 smithi189 bash[23614]: audit 2024-03-07T03:06:24.970396+0000 mon.a (mon.0) 697 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:25.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:25 smithi189 bash[23614]: audit 2024-03-07T03:06:24.990525+0000 mon.a (mon.0) 698 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:25.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:25 smithi189 bash[23614]: audit 2024-03-07T03:06:24.994728+0000 mon.a (mon.0) 699 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:25.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:25 smithi189 bash[23614]: audit 2024-03-07T03:06:25.043601+0000 mon.c (mon.2) 41 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:26.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:26 smithi145 bash[21339]: cluster 2024-03-07T03:06:26.362201+0000 mgr.y (mgr.24302) 33 : cluster 0 pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:26.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:26 smithi145 bash[30906]: cluster 2024-03-07T03:06:26.362201+0000 mgr.y (mgr.24302) 33 : cluster 0 pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:26.568 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:26 smithi145 bash[56172]: ts=2024-03-07T03:06:26.271Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-03-07T03:06:26.568 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:26 smithi145 bash[56172]: ts=2024-03-07T03:06:26.271Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-03-07T03:06:26.568 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:26 smithi145 bash[56172]: ts=2024-03-07T03:06:26.273Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.145 port=9094 2024-03-07T03:06:26.568 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:26 smithi145 bash[56172]: ts=2024-03-07T03:06:26.274Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-03-07T03:06:26.568 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:26 smithi145 bash[56172]: ts=2024-03-07T03:06:26.304Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-07T03:06:26.568 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:26 smithi145 bash[56172]: ts=2024-03-07T03:06:26.305Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-07T03:06:26.568 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:26 smithi145 bash[56172]: ts=2024-03-07T03:06:26.308Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-03-07T03:06:26.568 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:26 smithi145 bash[56172]: ts=2024-03-07T03:06:26.308Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-03-07T03:06:26.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:26 smithi189 bash[23614]: cluster 2024-03-07T03:06:26.362201+0000 mgr.y (mgr.24302) 33 : cluster 0 pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:28.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:28 smithi145 bash[21339]: audit 2024-03-07T03:06:27.449053+0000 mon.a (mon.0) 700 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:28.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:28 smithi145 bash[21339]: cluster 2024-03-07T03:06:28.362723+0000 mgr.y (mgr.24302) 34 : cluster 0 pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:28.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:28 smithi145 bash[30906]: audit 2024-03-07T03:06:27.449053+0000 mon.a (mon.0) 700 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:28.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:28 smithi145 bash[30906]: cluster 2024-03-07T03:06:28.362723+0000 mgr.y (mgr.24302) 34 : cluster 0 pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:28.568 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:28 smithi145 bash[56172]: ts=2024-03-07T03:06:28.274Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000614288s 2024-03-07T03:06:28.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:28 smithi189 bash[23614]: audit 2024-03-07T03:06:27.449053+0000 mon.a (mon.0) 700 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:28.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:28 smithi189 bash[23614]: cluster 2024-03-07T03:06:28.362723+0000 mgr.y (mgr.24302) 34 : cluster 0 pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:29.255 INFO:teuthology.orchestra.run.smithi189.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:06:31.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:30 smithi189 bash[23614]: audit 2024-03-07T03:06:29.835901+0000 mon.a (mon.0) 701 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:31.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:30 smithi189 bash[23614]: audit 2024-03-07T03:06:29.851075+0000 mon.a (mon.0) 702 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:31.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:30 smithi189 bash[23614]: cluster 2024-03-07T03:06:30.363332+0000 mgr.y (mgr.24302) 35 : cluster 0 pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:31.317 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:30 smithi145 bash[21339]: audit 2024-03-07T03:06:29.835901+0000 mon.a (mon.0) 701 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:31.317 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:30 smithi145 bash[21339]: audit 2024-03-07T03:06:29.851075+0000 mon.a (mon.0) 702 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:31.317 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:30 smithi145 bash[21339]: cluster 2024-03-07T03:06:30.363332+0000 mgr.y (mgr.24302) 35 : cluster 0 pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:31.317 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:30 smithi145 bash[30906]: audit 2024-03-07T03:06:29.835901+0000 mon.a (mon.0) 701 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:31.317 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:30 smithi145 bash[30906]: audit 2024-03-07T03:06:29.851075+0000 mon.a (mon.0) 702 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:31.317 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:30 smithi145 bash[30906]: cluster 2024-03-07T03:06:30.363332+0000 mgr.y (mgr.24302) 35 : cluster 0 pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:31.991 INFO:teuthology.orchestra.run.smithi189.stdout:[client.1] 2024-03-07T03:06:31.992 INFO:teuthology.orchestra.run.smithi189.stdout: key = AQA3L+llK3PnOhAAZh71HmHQeU5b7vfiHai8kA== 2024-03-07T03:06:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:31 smithi145 bash[21623]: [07/Mar/2024:03:06:31] ENGINE Bus STOPPING 2024-03-07T03:06:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:31 smithi145 bash[21623]: [07/Mar/2024:03:06:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:06:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:31 smithi145 bash[21623]: [07/Mar/2024:03:06:31] ENGINE Bus STOPPED 2024-03-07T03:06:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:31 smithi145 bash[21623]: [07/Mar/2024:03:06:31] ENGINE Bus STARTING 2024-03-07T03:06:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:31 smithi145 bash[21623]: [07/Mar/2024:03:06:31] ENGINE Serving on http://:::9283 2024-03-07T03:06:32.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:31 smithi145 bash[21623]: [07/Mar/2024:03:06:31] ENGINE Bus STARTED 2024-03-07T03:06:32.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: audit 2024-03-07T03:06:31.438339+0000 mon.a (mon.0) 703 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: audit 2024-03-07T03:06:31.453573+0000 mon.a (mon.0) 704 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: audit 2024-03-07T03:06:31.456245+0000 mon.c (mon.2) 42 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:32.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: audit 2024-03-07T03:06:31.457145+0000 mon.c (mon.2) 43 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:32.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: audit 2024-03-07T03:06:31.466439+0000 mon.a (mon.0) 705 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: cephadm 2024-03-07T03:06:31.493006+0000 mgr.y (mgr.24302) 36 : cephadm 1 Regenerating cephadm self-signed grafana TLS certificates 2024-03-07T03:06:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: audit 2024-03-07T03:06:31.542010+0000 mon.a (mon.0) 706 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: audit 2024-03-07T03:06:31.552027+0000 mon.a (mon.0) 707 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: audit 2024-03-07T03:06:31.558797+0000 mon.c (mon.2) 44 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-07T03:06:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: audit 2024-03-07T03:06:31.560008+0000 mgr.y (mgr.24302) 37 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-07T03:06:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: audit 2024-03-07T03:06:31.569236+0000 mon.a (mon.0) 708 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: cephadm 2024-03-07T03:06:31.584259+0000 mgr.y (mgr.24302) 38 : cephadm 1 Deploying daemon grafana.a on smithi189 2024-03-07T03:06:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: audit 2024-03-07T03:06:31.987157+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.189:0/2085133695' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:06:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: audit 2024-03-07T03:06:31.987920+0000 mon.a (mon.0) 709 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:06:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: audit 2024-03-07T03:06:31.995831+0000 mon.a (mon.0) 710 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-07T03:06:32.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:32 smithi189 bash[23614]: cluster 2024-03-07T03:06:32.363939+0000 mgr.y (mgr.24302) 39 : cluster 0 pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:32.618 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-03-07T03:06:32.618 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-03-07T03:06:32.618 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-03-07T03:06:32.642 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-03-07T03:06:32.642 DEBUG:teuthology.orchestra.run.smithi145:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-03-07T03:06:32.817 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:06:32] "GET /metrics HTTP/1.1" 200 26212 "" "Prometheus/2.43.0" 2024-03-07T03:06:32.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: audit 2024-03-07T03:06:31.438339+0000 mon.a (mon.0) 703 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: audit 2024-03-07T03:06:31.453573+0000 mon.a (mon.0) 704 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: audit 2024-03-07T03:06:31.456245+0000 mon.c (mon.2) 42 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:32.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: audit 2024-03-07T03:06:31.457145+0000 mon.c (mon.2) 43 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:32.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: audit 2024-03-07T03:06:31.466439+0000 mon.a (mon.0) 705 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: cephadm 2024-03-07T03:06:31.493006+0000 mgr.y (mgr.24302) 36 : cephadm 1 Regenerating cephadm self-signed grafana TLS certificates 2024-03-07T03:06:32.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: audit 2024-03-07T03:06:31.542010+0000 mon.a (mon.0) 706 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: audit 2024-03-07T03:06:31.552027+0000 mon.a (mon.0) 707 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: audit 2024-03-07T03:06:31.558797+0000 mon.c (mon.2) 44 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-07T03:06:32.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: audit 2024-03-07T03:06:31.560008+0000 mgr.y (mgr.24302) 37 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-07T03:06:32.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: audit 2024-03-07T03:06:31.569236+0000 mon.a (mon.0) 708 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: cephadm 2024-03-07T03:06:31.584259+0000 mgr.y (mgr.24302) 38 : cephadm 1 Deploying daemon grafana.a on smithi189 2024-03-07T03:06:32.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: audit 2024-03-07T03:06:31.987157+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.189:0/2085133695' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:06:32.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: audit 2024-03-07T03:06:31.987920+0000 mon.a (mon.0) 709 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:06:32.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: audit 2024-03-07T03:06:31.995831+0000 mon.a (mon.0) 710 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-07T03:06:32.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[21339]: cluster 2024-03-07T03:06:32.363939+0000 mgr.y (mgr.24302) 39 : cluster 0 pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:32.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: audit 2024-03-07T03:06:31.438339+0000 mon.a (mon.0) 703 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: audit 2024-03-07T03:06:31.453573+0000 mon.a (mon.0) 704 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: audit 2024-03-07T03:06:31.456245+0000 mon.c (mon.2) 42 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: audit 2024-03-07T03:06:31.457145+0000 mon.c (mon.2) 43 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: audit 2024-03-07T03:06:31.466439+0000 mon.a (mon.0) 705 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: cephadm 2024-03-07T03:06:31.493006+0000 mgr.y (mgr.24302) 36 : cephadm 1 Regenerating cephadm self-signed grafana TLS certificates 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: audit 2024-03-07T03:06:31.542010+0000 mon.a (mon.0) 706 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: audit 2024-03-07T03:06:31.552027+0000 mon.a (mon.0) 707 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: audit 2024-03-07T03:06:31.558797+0000 mon.c (mon.2) 44 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: audit 2024-03-07T03:06:31.560008+0000 mgr.y (mgr.24302) 37 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: audit 2024-03-07T03:06:31.569236+0000 mon.a (mon.0) 708 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: cephadm 2024-03-07T03:06:31.584259+0000 mgr.y (mgr.24302) 38 : cephadm 1 Deploying daemon grafana.a on smithi189 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: audit 2024-03-07T03:06:31.987157+0000 mon.b (mon.1) 30 : audit 1 from='client.? 172.21.15.189:0/2085133695' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: audit 2024-03-07T03:06:31.987920+0000 mon.a (mon.0) 709 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: audit 2024-03-07T03:06:31.995831+0000 mon.a (mon.0) 710 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-07T03:06:32.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:32 smithi145 bash[30906]: cluster 2024-03-07T03:06:32.363939+0000 mgr.y (mgr.24302) 39 : cluster 0 pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:32.951 INFO:teuthology.misc.health.smithi145.stdout: 2024-03-07T03:06:32.951 INFO:teuthology.misc.health.smithi145.stdout:{"epoch":60,"fsid":"8947a0f8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:55:42.039360+0000","modified":"2024-03-07T03:06:02.331079+0000","last_up_change":"2024-03-07T03:05:04.610834+0000","last_in_change":"2024-03-07T03:04:39.932769+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:00:59.846304+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"242c63ee-0417-4ebf-bd71-6239f5d3634f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6803","nonce":1367316946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6805","nonce":1367316946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6809","nonce":1367316946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6807","nonce":1367316946}]},"public_addr":"172.21.15.145:6803/1367316946","cluster_addr":"172.21.15.145:6805/1367316946","heartbeat_back_addr":"172.21.15.145:6809/1367316946","heartbeat_front_addr":"172.21.15.145:6807/1367316946","state":["exists","up"]},{"osd":1,"uuid":"3c2818da-5d32-4f7a-8a63-8d7d679961e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6811","nonce":1873038359}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6813","nonce":1873038359}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6817","nonce":1873038359}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6815","nonce":1873038359}]},"public_addr":"172.21.15.145:6811/1873038359","cluster_addr":"172.21.15.145:6813/1873038359","heartbeat_back_addr":"172.21.15.145:6817/1873038359","heartbeat_front_addr":"172.21.15.145:6815/1873038359","state":["exists","up"]},{"osd":2,"uuid":"905ec219-dd9c-4f30-bdab-ed78d1405c11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6819","nonce":462216015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6821","nonce":462216015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6825","nonce":462216015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6823","nonce":462216015}]},"public_addr":"172.21.15.145:6819/462216015","cluster_addr":"172.21.15.145:6821/462216015","heartbeat_back_addr":"172.21.15.145:6825/462216015","heartbeat_front_addr":"172.21.15.145:6823/462216015","state":["exists","up"]},{"osd":3,"uuid":"5f466751-f5a7-4d18-87ab-96251b77b663","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6827","nonce":2306759084}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6829","nonce":2306759084}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6833","nonce":2306759084}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6831","nonce":2306759084}]},"public_addr":"172.21.15.145:6827/2306759084","cluster_addr":"172.21.15.145:6829/2306759084","heartbeat_back_addr":"172.21.15.145:6833/2306759084","heartbeat_front_addr":"172.21.15.145:6831/2306759084","state":["exists","up"]},{"osd":4,"uuid":"633f56af-3ff8-4954-97b4-bc8122bb10ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6801","nonce":3152992669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6803","nonce":3152992669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6807","nonce":3152992669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6805","nonce":3152992669}]},"public_addr":"172.21.15.189:6801/3152992669","cluster_addr":"172.21.15.189:6803/3152992669","heartbeat_back_addr":"172.21.15.189:6807/3152992669","heartbeat_front_addr":"172.21.15.189:6805/3152992669","state":["exists","up"]},{"osd":5,"uuid":"531d5127-60fe-4669-ab7d-fe4f5e5753cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6809","nonce":2096444201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6811","nonce":2096444201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6815","nonce":2096444201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6813","nonce":2096444201}]},"public_addr":"172.21.15.189:6809/2096444201","cluster_addr":"172.21.15.189:6811/2096444201","heartbeat_back_addr":"172.21.15.189:6815/2096444201","heartbeat_front_addr":"172.21.15.189:6813/2096444201","state":["exists","up"]},{"osd":6,"uuid":"8f2b456e-712e-45f1-8d84-0fdc4a6d860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6817","nonce":4219543665}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6819","nonce":4219543665}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6823","nonce":4219543665}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6821","nonce":4219543665}]},"public_addr":"172.21.15.189:6817/4219543665","cluster_addr":"172.21.15.189:6819/4219543665","heartbeat_back_addr":"172.21.15.189:6823/4219543665","heartbeat_front_addr":"172.21.15.189:6821/4219543665","state":["exists","up"]},{"osd":7,"uuid":"24dc7eba-4cd4-414d-93d2-598f5f2e655c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6825","nonce":452348965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6827","nonce":452348965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6831","nonce":452348965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6829","nonce":452348965}]},"public_addr":"172.21.15.189:6825/452348965","cluster_addr":"172.21.15.189:6827/452348965","heartbeat_back_addr":"172.21.15.189:6831/452348965","heartbeat_front_addr":"172.21.15.189:6829/452348965","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T02:59:11.112897+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:03.501748+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:56.635004+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:01:50.372735+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:38.150721+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:25.965239+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:14.345171+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:03.479901+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.145:6801/2891889777":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3299433409":"2024-03-08T03:06:02.331013+0000","172.21.15.145:6800/2891889777":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/2939737297":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/3852118874":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3488231986":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/2236140381":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/1978862755":"2024-03-08T03:06:02.331013+0000","172.21.15.145:6800/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/2696605014":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/131277059":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/2414597158":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:6800/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/321164949":"2024-03-08T02:56:14.659813+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:06:32.970 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-03-07T03:06:32.970 INFO:tasks.cephadm:Creating RBD pool 2024-03-07T03:06:32.970 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-03-07T03:06:33.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:33 smithi145 bash[21339]: audit 2024-03-07T03:06:32.447104+0000 mon.c (mon.2) 45 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:06:33.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:33 smithi145 bash[21339]: audit 2024-03-07T03:06:32.954732+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.145:0/189345731' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:06:33.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:33 smithi145 bash[30906]: audit 2024-03-07T03:06:32.447104+0000 mon.c (mon.2) 45 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:06:33.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:33 smithi145 bash[30906]: audit 2024-03-07T03:06:32.954732+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.145:0/189345731' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:06:33.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:33 smithi189 bash[23614]: audit 2024-03-07T03:06:32.447104+0000 mon.c (mon.2) 45 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:06:33.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:33 smithi189 bash[23614]: audit 2024-03-07T03:06:32.954732+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.145:0/189345731' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:06:34.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:34 smithi145 bash[21339]: cluster 2024-03-07T03:06:34.364674+0000 mgr.y (mgr.24302) 40 : cluster 0 pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:34.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:34 smithi145 bash[30906]: cluster 2024-03-07T03:06:34.364674+0000 mgr.y (mgr.24302) 40 : cluster 0 pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:34.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:34 smithi189 bash[23614]: cluster 2024-03-07T03:06:34.364674+0000 mgr.y (mgr.24302) 40 : cluster 0 pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:36.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:36 smithi145 bash[21339]: cluster 2024-03-07T03:06:36.365354+0000 mgr.y (mgr.24302) 41 : cluster 0 pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:36.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:36 smithi145 bash[30906]: cluster 2024-03-07T03:06:36.365354+0000 mgr.y (mgr.24302) 41 : cluster 0 pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:36.568 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:06:36 smithi145 bash[56172]: ts=2024-03-07T03:06:36.277Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003185298s 2024-03-07T03:06:36.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:36 smithi189 bash[23614]: cluster 2024-03-07T03:06:36.365354+0000 mgr.y (mgr.24302) 41 : cluster 0 pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:37.692 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:06:39.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:39 smithi145 bash[21339]: cluster 2024-03-07T03:06:38.366077+0000 mgr.y (mgr.24302) 42 : cluster 0 pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:39.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:39 smithi145 bash[30906]: cluster 2024-03-07T03:06:38.366077+0000 mgr.y (mgr.24302) 42 : cluster 0 pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:39.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:39 smithi189 bash[23614]: cluster 2024-03-07T03:06:38.366077+0000 mgr.y (mgr.24302) 42 : cluster 0 pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:40.757 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:40 smithi145 bash[21339]: cluster 2024-03-07T03:06:40.366702+0000 mgr.y (mgr.24302) 43 : cluster 0 pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:40.757 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:40 smithi145 bash[30906]: cluster 2024-03-07T03:06:40.366702+0000 mgr.y (mgr.24302) 43 : cluster 0 pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:40.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:40 smithi189 bash[23614]: cluster 2024-03-07T03:06:40.366702+0000 mgr.y (mgr.24302) 43 : cluster 0 pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:41.647 INFO:teuthology.orchestra.run.smithi145.stderr:pool 'rbd' created 2024-03-07T03:06:41.915 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:41 smithi145 bash[21339]: audit 2024-03-07T03:06:41.146857+0000 mon.a (mon.0) 711 : audit 1 from='client.? 172.21.15.145:0/2822174637' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-07T03:06:41.916 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:41 smithi145 bash[30906]: audit 2024-03-07T03:06:41.146857+0000 mon.a (mon.0) 711 : audit 1 from='client.? 172.21.15.145:0/2822174637' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-07T03:06:42.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:41 smithi189 bash[23614]: audit 2024-03-07T03:06:41.146857+0000 mon.a (mon.0) 711 : audit 1 from='client.? 172.21.15.145:0/2822174637' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-07T03:06:42.482 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-03-07T03:06:42.652 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:42 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:06:42] "GET /metrics HTTP/1.1" 200 26210 "" "Prometheus/2.43.0" 2024-03-07T03:06:42.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:42 smithi145 bash[21339]: audit 2024-03-07T03:06:41.646414+0000 mon.a (mon.0) 712 : audit 1 from='client.? 172.21.15.145:0/2822174637' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-07T03:06:42.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:42 smithi145 bash[21339]: cluster 2024-03-07T03:06:41.646612+0000 mon.a (mon.0) 713 : cluster 0 osdmap e61: 8 total, 8 up, 8 in 2024-03-07T03:06:42.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:42 smithi145 bash[21339]: cluster 2024-03-07T03:06:42.367314+0000 mgr.y (mgr.24302) 44 : cluster 0 pgmap v24: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:42.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:42 smithi145 bash[30906]: audit 2024-03-07T03:06:41.646414+0000 mon.a (mon.0) 712 : audit 1 from='client.? 172.21.15.145:0/2822174637' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-07T03:06:42.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:42 smithi145 bash[30906]: cluster 2024-03-07T03:06:41.646612+0000 mon.a (mon.0) 713 : cluster 0 osdmap e61: 8 total, 8 up, 8 in 2024-03-07T03:06:42.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:42 smithi145 bash[30906]: cluster 2024-03-07T03:06:42.367314+0000 mgr.y (mgr.24302) 44 : cluster 0 pgmap v24: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:43.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:42 smithi189 bash[23614]: audit 2024-03-07T03:06:41.646414+0000 mon.a (mon.0) 712 : audit 1 from='client.? 172.21.15.145:0/2822174637' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-07T03:06:43.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:42 smithi189 bash[23614]: cluster 2024-03-07T03:06:41.646612+0000 mon.a (mon.0) 713 : cluster 0 osdmap e61: 8 total, 8 up, 8 in 2024-03-07T03:06:43.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:42 smithi189 bash[23614]: cluster 2024-03-07T03:06:42.367314+0000 mgr.y (mgr.24302) 44 : cluster 0 pgmap v24: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:44.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:43 smithi145 bash[21339]: cluster 2024-03-07T03:06:42.650397+0000 mon.a (mon.0) 714 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:06:44.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:43 smithi145 bash[21339]: cluster 2024-03-07T03:06:42.660205+0000 mon.a (mon.0) 715 : cluster 0 osdmap e62: 8 total, 8 up, 8 in 2024-03-07T03:06:44.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:43 smithi145 bash[30906]: cluster 2024-03-07T03:06:42.650397+0000 mon.a (mon.0) 714 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:06:44.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:43 smithi145 bash[30906]: cluster 2024-03-07T03:06:42.660205+0000 mon.a (mon.0) 715 : cluster 0 osdmap e62: 8 total, 8 up, 8 in 2024-03-07T03:06:44.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:43 smithi189 bash[23614]: cluster 2024-03-07T03:06:42.650397+0000 mon.a (mon.0) 714 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:06:44.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:43 smithi189 bash[23614]: cluster 2024-03-07T03:06:42.660205+0000 mon.a (mon.0) 715 : cluster 0 osdmap e62: 8 total, 8 up, 8 in 2024-03-07T03:06:45.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:44 smithi145 bash[21339]: cluster 2024-03-07T03:06:43.669949+0000 mon.a (mon.0) 716 : cluster 0 osdmap e63: 8 total, 8 up, 8 in 2024-03-07T03:06:45.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:44 smithi145 bash[30906]: cluster 2024-03-07T03:06:43.669949+0000 mon.a (mon.0) 716 : cluster 0 osdmap e63: 8 total, 8 up, 8 in 2024-03-07T03:06:45.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:44 smithi189 bash[23614]: cluster 2024-03-07T03:06:43.669949+0000 mon.a (mon.0) 716 : cluster 0 osdmap e63: 8 total, 8 up, 8 in 2024-03-07T03:06:46.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:45 smithi145 bash[21339]: cluster 2024-03-07T03:06:44.368014+0000 mgr.y (mgr.24302) 45 : cluster 0 pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:46.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:45 smithi145 bash[30906]: cluster 2024-03-07T03:06:44.368014+0000 mgr.y (mgr.24302) 45 : cluster 0 pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:46.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:45 smithi189 bash[23614]: cluster 2024-03-07T03:06:44.368014+0000 mgr.y (mgr.24302) 45 : cluster 0 pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:47.206 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:06:48.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:47 smithi145 bash[21339]: cluster 2024-03-07T03:06:46.368717+0000 mgr.y (mgr.24302) 46 : cluster 0 pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:48.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:47 smithi145 bash[21339]: audit 2024-03-07T03:06:47.447727+0000 mon.a (mon.0) 717 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:48.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:47 smithi145 bash[21339]: audit 2024-03-07T03:06:47.449798+0000 mon.c (mon.2) 47 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:06:48.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:47 smithi145 bash[30906]: cluster 2024-03-07T03:06:46.368717+0000 mgr.y (mgr.24302) 46 : cluster 0 pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:48.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:47 smithi145 bash[30906]: audit 2024-03-07T03:06:47.447727+0000 mon.a (mon.0) 717 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:48.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:47 smithi145 bash[30906]: audit 2024-03-07T03:06:47.449798+0000 mon.c (mon.2) 47 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:06:48.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:47 smithi189 bash[23614]: cluster 2024-03-07T03:06:46.368717+0000 mgr.y (mgr.24302) 46 : cluster 0 pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:48.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:47 smithi189 bash[23614]: audit 2024-03-07T03:06:47.447727+0000 mon.a (mon.0) 717 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:48.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:47 smithi189 bash[23614]: audit 2024-03-07T03:06:47.449798+0000 mon.c (mon.2) 47 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:06:50.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:50 smithi145 bash[21339]: cluster 2024-03-07T03:06:48.369142+0000 mgr.y (mgr.24302) 47 : cluster 0 pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:50.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:50 smithi145 bash[30906]: cluster 2024-03-07T03:06:48.369142+0000 mgr.y (mgr.24302) 47 : cluster 0 pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:50.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:50 smithi189 bash[23614]: cluster 2024-03-07T03:06:48.369142+0000 mgr.y (mgr.24302) 47 : cluster 0 pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:51.329 INFO:teuthology.orchestra.run.smithi145.stderr:enabled application 'rbd' on pool 'rbd' 2024-03-07T03:06:51.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:51 smithi145 bash[21339]: audit 2024-03-07T03:06:50.426100+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.145:0/3189961768' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:06:51.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:51 smithi145 bash[21339]: audit 2024-03-07T03:06:50.426738+0000 mon.a (mon.0) 718 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:06:51.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:51 smithi145 bash[30906]: audit 2024-03-07T03:06:50.426100+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.145:0/3189961768' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:06:51.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:51 smithi145 bash[30906]: audit 2024-03-07T03:06:50.426738+0000 mon.a (mon.0) 718 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:06:51.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:51 smithi189 bash[23614]: audit 2024-03-07T03:06:50.426100+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.145:0/3189961768' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:06:51.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:51 smithi189 bash[23614]: audit 2024-03-07T03:06:50.426738+0000 mon.a (mon.0) 718 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:06:52.077 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-07T03:06:52.077 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-07T03:06:52.077 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph mgr dump --format=json 2024-03-07T03:06:52.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:52 smithi189 bash[23614]: cluster 2024-03-07T03:06:50.369579+0000 mgr.y (mgr.24302) 48 : cluster 0 pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:52.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:52 smithi189 bash[23614]: audit 2024-03-07T03:06:51.327540+0000 mon.a (mon.0) 719 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-07T03:06:52.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:52 smithi189 bash[23614]: cluster 2024-03-07T03:06:51.327702+0000 mon.a (mon.0) 720 : cluster 0 osdmap e64: 8 total, 8 up, 8 in 2024-03-07T03:06:52.816 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:06:52 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:06:52] "GET /metrics HTTP/1.1" 200 28552 "" "Prometheus/2.43.0" 2024-03-07T03:06:52.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:52 smithi145 bash[21339]: cluster 2024-03-07T03:06:50.369579+0000 mgr.y (mgr.24302) 48 : cluster 0 pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:52.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:52 smithi145 bash[21339]: audit 2024-03-07T03:06:51.327540+0000 mon.a (mon.0) 719 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-07T03:06:52.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:52 smithi145 bash[21339]: cluster 2024-03-07T03:06:51.327702+0000 mon.a (mon.0) 720 : cluster 0 osdmap e64: 8 total, 8 up, 8 in 2024-03-07T03:06:52.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:52 smithi145 bash[30906]: cluster 2024-03-07T03:06:50.369579+0000 mgr.y (mgr.24302) 48 : cluster 0 pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:52.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:52 smithi145 bash[30906]: audit 2024-03-07T03:06:51.327540+0000 mon.a (mon.0) 719 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-07T03:06:52.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:52 smithi145 bash[30906]: cluster 2024-03-07T03:06:51.327702+0000 mon.a (mon.0) 720 : cluster 0 osdmap e64: 8 total, 8 up, 8 in 2024-03-07T03:06:54.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:53 smithi145 bash[21339]: cluster 2024-03-07T03:06:52.370115+0000 mgr.y (mgr.24302) 49 : cluster 0 pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:54.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:53 smithi145 bash[21339]: audit 2024-03-07T03:06:52.798438+0000 mon.a (mon.0) 721 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:54.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:53 smithi145 bash[21339]: cluster 2024-03-07T03:06:53.331912+0000 mon.a (mon.0) 722 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:06:54.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:53 smithi145 bash[21339]: cluster 2024-03-07T03:06:53.331963+0000 mon.a (mon.0) 723 : cluster 1 Cluster is now healthy 2024-03-07T03:06:54.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:53 smithi145 bash[30906]: cluster 2024-03-07T03:06:52.370115+0000 mgr.y (mgr.24302) 49 : cluster 0 pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:54.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:53 smithi145 bash[30906]: audit 2024-03-07T03:06:52.798438+0000 mon.a (mon.0) 721 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:54.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:53 smithi145 bash[30906]: cluster 2024-03-07T03:06:53.331912+0000 mon.a (mon.0) 722 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:06:54.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:53 smithi145 bash[30906]: cluster 2024-03-07T03:06:53.331963+0000 mon.a (mon.0) 723 : cluster 1 Cluster is now healthy 2024-03-07T03:06:54.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:53 smithi189 bash[23614]: cluster 2024-03-07T03:06:52.370115+0000 mgr.y (mgr.24302) 49 : cluster 0 pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:54.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:53 smithi189 bash[23614]: audit 2024-03-07T03:06:52.798438+0000 mon.a (mon.0) 721 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:54.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:53 smithi189 bash[23614]: cluster 2024-03-07T03:06:53.331912+0000 mon.a (mon.0) 722 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:06:54.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:53 smithi189 bash[23614]: cluster 2024-03-07T03:06:53.331963+0000 mon.a (mon.0) 723 : cluster 1 Cluster is now healthy 2024-03-07T03:06:55.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:55 smithi189 bash[23614]: cluster 2024-03-07T03:06:54.370762+0000 mgr.y (mgr.24302) 50 : cluster 0 pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:56.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:55 smithi145 bash[21339]: cluster 2024-03-07T03:06:54.370762+0000 mgr.y (mgr.24302) 50 : cluster 0 pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:56.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:55 smithi145 bash[30906]: cluster 2024-03-07T03:06:54.370762+0000 mgr.y (mgr.24302) 50 : cluster 0 pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:56.769 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:56.769 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:56.770 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:56.770 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:56.770 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:56.770 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:56.771 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:56.771 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:56.801 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:06:57.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:57.087 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:57.087 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:57.087 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:57.088 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:57.088 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:57.088 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:57.088 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:57.089 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:56 smithi189 systemd[1]: Started Ceph grafana.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:06:57.927 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.886773433Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-03-07T03:06:57.927 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.886984155Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.887006181Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.887018716Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.887026312Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.887033419Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.887040094Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.887047904Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.887057872Z level=info msg="Path Home" path=/usr/share/grafana 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.887068782Z level=info msg="Path Data" path=/var/lib/grafana 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.8870811Z level=info msg="Path Logs" path=/var/log/grafana 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.887095305Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.887102439Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=settings t=2024-03-07T03:06:57.887109639Z level=info msg="App mode production" 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=sqlstore t=2024-03-07T03:06:57.887169422Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-03-07T03:06:57.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=sqlstore t=2024-03-07T03:06:57.887195282Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-03-07T03:06:57.929 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:57.888180412Z level=info msg="Starting DB migrations" 2024-03-07T03:06:57.929 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:57.888273661Z level=info msg="Executing migration" id="create migration_log table" 2024-03-07T03:06:58.183 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:57.93588988Z level=info msg="Executing migration" id="create user table" 2024-03-07T03:06:58.184 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:57.986819735Z level=info msg="Executing migration" id="add unique index user.login" 2024-03-07T03:06:58.184 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.012133999Z level=info msg="Executing migration" id="add unique index user.email" 2024-03-07T03:06:58.184 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.037382944Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-03-07T03:06:58.184 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.079285574Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-03-07T03:06:58.184 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.105076428Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-03-07T03:06:58.184 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.129728052Z level=info msg="Executing migration" id="create user table v2" 2024-03-07T03:06:58.184 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.171715954Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-03-07T03:06:58.184 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.188678778Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-03-07T03:06:58.184 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[23614]: cluster 2024-03-07T03:06:56.371358+0000 mgr.y (mgr.24302) 51 : cluster 0 pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:58.184 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[23614]: audit 2024-03-07T03:06:56.934828+0000 mon.a (mon.0) 724 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.184 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[23614]: audit 2024-03-07T03:06:56.953900+0000 mon.a (mon.0) 725 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.184 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[23614]: audit 2024-03-07T03:06:57.018573+0000 mon.a (mon.0) 726 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.185 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[23614]: audit 2024-03-07T03:06:57.029564+0000 mon.a (mon.0) 727 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.185 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[23614]: audit 2024-03-07T03:06:57.098529+0000 mon.c (mon.2) 49 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:58.185 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[23614]: audit 2024-03-07T03:06:57.474526+0000 mon.a (mon.0) 728 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.185 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[23614]: audit 2024-03-07T03:06:57.482844+0000 mon.a (mon.0) 729 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.185 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:06:57 smithi189 bash[23614]: audit 2024-03-07T03:06:57.812367+0000 mon.a (mon.0) 730 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.263 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[21339]: cluster 2024-03-07T03:06:56.371358+0000 mgr.y (mgr.24302) 51 : cluster 0 pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:58.263 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[21339]: audit 2024-03-07T03:06:56.934828+0000 mon.a (mon.0) 724 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.263 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[21339]: audit 2024-03-07T03:06:56.953900+0000 mon.a (mon.0) 725 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.263 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[21339]: audit 2024-03-07T03:06:57.018573+0000 mon.a (mon.0) 726 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.263 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[21339]: audit 2024-03-07T03:06:57.029564+0000 mon.a (mon.0) 727 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.263 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[21339]: audit 2024-03-07T03:06:57.098529+0000 mon.c (mon.2) 49 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:58.263 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[21339]: audit 2024-03-07T03:06:57.474526+0000 mon.a (mon.0) 728 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.263 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[21339]: audit 2024-03-07T03:06:57.482844+0000 mon.a (mon.0) 729 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.263 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[21339]: audit 2024-03-07T03:06:57.812367+0000 mon.a (mon.0) 730 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.263 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[30906]: cluster 2024-03-07T03:06:56.371358+0000 mgr.y (mgr.24302) 51 : cluster 0 pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:06:58.264 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[30906]: audit 2024-03-07T03:06:56.934828+0000 mon.a (mon.0) 724 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.264 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[30906]: audit 2024-03-07T03:06:56.953900+0000 mon.a (mon.0) 725 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.264 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[30906]: audit 2024-03-07T03:06:57.018573+0000 mon.a (mon.0) 726 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.264 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[30906]: audit 2024-03-07T03:06:57.029564+0000 mon.a (mon.0) 727 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.264 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[30906]: audit 2024-03-07T03:06:57.098529+0000 mon.c (mon.2) 49 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:58.264 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[30906]: audit 2024-03-07T03:06:57.474526+0000 mon.a (mon.0) 728 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.264 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[30906]: audit 2024-03-07T03:06:57.482844+0000 mon.a (mon.0) 729 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.264 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:06:57 smithi145 bash[30906]: audit 2024-03-07T03:06:57.812367+0000 mon.a (mon.0) 730 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:06:58.461 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.205595829Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-03-07T03:06:58.461 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.222343831Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-03-07T03:06:58.461 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.239335162Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-03-07T03:06:58.461 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.272875924Z level=info msg="Executing migration" id="Update user table charset" 2024-03-07T03:06:58.461 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.289770097Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-03-07T03:06:58.461 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.315106242Z level=info msg="Executing migration" id="Add missing user data" 2024-03-07T03:06:58.461 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.356970819Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-03-07T03:06:58.462 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.382297677Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-03-07T03:06:58.462 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.424235001Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-03-07T03:06:58.462 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.466404462Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-03-07T03:06:58.722 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.508427706Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-03-07T03:06:58.722 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.533688898Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-03-07T03:06:58.722 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.559000609Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-03-07T03:06:58.722 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.58416287Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-03-07T03:06:58.722 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.626170138Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-03-07T03:06:58.722 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.65141637Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-03-07T03:06:58.722 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.693316476Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-03-07T03:06:58.722 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.72696563Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-03-07T03:06:58.984 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.752181706Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-03-07T03:06:58.985 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.777465624Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-03-07T03:06:58.985 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.819422863Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-03-07T03:06:58.985 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.861668801Z level=info msg="Executing migration" id="create temp_user v2" 2024-03-07T03:06:58.985 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.903622556Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-03-07T03:06:58.985 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.945621331Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-03-07T03:06:58.985 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:58 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:58.989566861Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-03-07T03:06:59.276 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.012792296Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-03-07T03:06:59.276 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.038193087Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-03-07T03:06:59.276 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.063322051Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-03-07T03:06:59.277 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.088528591Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-03-07T03:06:59.277 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.147121119Z level=info msg="Executing migration" id="create star table" 2024-03-07T03:06:59.277 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.189123822Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-03-07T03:06:59.277 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.214408941Z level=info msg="Executing migration" id="create org table v1" 2024-03-07T03:06:59.277 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.239601994Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-03-07T03:06:59.277 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.281565731Z level=info msg="Executing migration" id="create org_user table v1" 2024-03-07T03:06:59.537 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.3068538Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-03-07T03:06:59.537 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.348686425Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-03-07T03:06:59.537 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.374005837Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-03-07T03:06:59.537 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.399272975Z level=info msg="Executing migration" id="Update org table charset" 2024-03-07T03:06:59.537 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.424453566Z level=info msg="Executing migration" id="Update org_user table charset" 2024-03-07T03:06:59.538 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.449829127Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-03-07T03:06:59.538 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.474879995Z level=info msg="Executing migration" id="create dashboard table" 2024-03-07T03:06:59.538 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.500269323Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-03-07T03:06:59.538 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.54222317Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-03-07T03:06:59.797 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.584316934Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-03-07T03:06:59.798 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.609617699Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-03-07T03:06:59.798 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.65153236Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-03-07T03:06:59.798 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.676783939Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-03-07T03:06:59.798 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.718809238Z level=info msg="Executing migration" id="create dashboard v2" 2024-03-07T03:06:59.798 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.77754084Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-03-07T03:06:59.798 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.80277054Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-03-07T03:07:00.050 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.844705257Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-03-07T03:07:00.050 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.870164437Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-03-07T03:07:00.050 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.912141953Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-03-07T03:07:00.050 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.95408002Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-03-07T03:07:00.050 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:06:59 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:06:59.979377729Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-03-07T03:07:00.050 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.004346518Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-03-07T03:07:00.050 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.021454818Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-03-07T03:07:00.050 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.038124902Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-03-07T03:07:00.050 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.055259876Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-03-07T03:07:00.153 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:07:00.318 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.072064316Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-03-07T03:07:00.318 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.08897765Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-03-07T03:07:00.318 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.10572562Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-03-07T03:07:00.318 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.139569535Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-03-07T03:07:00.318 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.165005002Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-03-07T03:07:00.318 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.190100031Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-03-07T03:07:00.318 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.232137071Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-03-07T03:07:00.318 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.25741164Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-03-07T03:07:00.318 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.299362678Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-03-07T03:07:00.420 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:00 smithi145 bash[21339]: cluster 2024-03-07T03:06:58.371968+0000 mgr.y (mgr.24302) 52 : cluster 0 pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:00.420 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:00 smithi145 bash[21339]: audit 2024-03-07T03:07:00.148766+0000 mon.a (mon.0) 731 : audit 0 from='client.? 172.21.15.145:0/2528883841' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:07:00.421 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:00 smithi145 bash[30906]: cluster 2024-03-07T03:06:58.371968+0000 mgr.y (mgr.24302) 52 : cluster 0 pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:00.421 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:00 smithi145 bash[30906]: audit 2024-03-07T03:07:00.148766+0000 mon.a (mon.0) 731 : audit 0 from='client.? 172.21.15.145:0/2528883841' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:07:00.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[23614]: cluster 2024-03-07T03:06:58.371968+0000 mgr.y (mgr.24302) 52 : cluster 0 pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:00.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[23614]: audit 2024-03-07T03:07:00.148766+0000 mon.a (mon.0) 731 : audit 0 from='client.? 172.21.15.145:0/2528883841' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:07:00.580 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.341410937Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-03-07T03:07:00.581 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.366752546Z level=info msg="Executing migration" id="Update dashboard title length" 2024-03-07T03:07:00.581 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.39194079Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-03-07T03:07:00.581 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.417234433Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-03-07T03:07:00.581 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.442448536Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-03-07T03:07:00.581 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.467648187Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-03-07T03:07:00.581 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.492951933Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-03-07T03:07:00.581 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.518190296Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-03-07T03:07:00.581 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.560126884Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-03-07T03:07:00.581 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.585391404Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-03-07T03:07:00.836 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.610583315Z level=info msg="Executing migration" id="Add check_sum column" 2024-03-07T03:07:00.836 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.652470804Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-03-07T03:07:00.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.694448949Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-03-07T03:07:00.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.719765987Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-03-07T03:07:00.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.753532622Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-03-07T03:07:00.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.778850046Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-03-07T03:07:00.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.820867161Z level=info msg="Executing migration" id="create data_source table" 2024-03-07T03:07:01.003 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":20,"active_gid":24302,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6800","nonce":1684821814},{"type":"v1","addr":"172.21.15.145:6801","nonce":1684821814}]},"active_addr":"172.21.15.145:6801/1684821814","active_change":"2024-03-07T03:06:02.331328+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24304,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.145:8443/","prometheus":"http://172.21.15.145:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":3349297031}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":1139507458}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":1420640097}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":4002816623}]}]} 2024-03-07T03:07:01.007 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-07T03:07:01.007 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-07T03:07:01.007 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:07:01.143 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.862790449Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-03-07T03:07:01.143 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.887947967Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-03-07T03:07:01.143 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.930040716Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-03-07T03:07:01.144 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.955316466Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-03-07T03:07:01.144 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:00 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:00.980528717Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-03-07T03:07:01.144 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.005680174Z level=info msg="Executing migration" id="create data_source table v2" 2024-03-07T03:07:01.144 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.030989098Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-03-07T03:07:01.144 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.056193692Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-03-07T03:07:01.144 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.081473965Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-03-07T03:07:01.144 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.106701177Z level=info msg="Executing migration" id="Add column with_credentials" 2024-03-07T03:07:01.144 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.148646235Z level=info msg="Executing migration" id="Add secure json data column" 2024-03-07T03:07:01.396 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.190599013Z level=info msg="Executing migration" id="Update data_source table charset" 2024-03-07T03:07:01.396 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.21587074Z level=info msg="Executing migration" id="Update initial version to 1" 2024-03-07T03:07:01.396 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.241129855Z level=info msg="Executing migration" id="Add read_only data column" 2024-03-07T03:07:01.396 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.266343089Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-03-07T03:07:01.396 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.308239565Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-03-07T03:07:01.396 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.350246829Z level=info msg="Executing migration" id="Add uid column" 2024-03-07T03:07:01.396 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.375465307Z level=info msg="Executing migration" id="Update uid value" 2024-03-07T03:07:01.396 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.400716283Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-03-07T03:07:01.656 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.425916482Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-03-07T03:07:01.656 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.451185662Z level=info msg="Executing migration" id="create api_key table" 2024-03-07T03:07:01.656 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.476446129Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-03-07T03:07:01.656 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.518422989Z level=info msg="Executing migration" id="add index api_key.key" 2024-03-07T03:07:01.656 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.535290578Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-03-07T03:07:01.656 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.568924533Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-03-07T03:07:01.656 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.594050912Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-03-07T03:07:01.657 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.636297659Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-03-07T03:07:01.657 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.661271325Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-03-07T03:07:01.951 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.711970174Z level=info msg="Executing migration" id="create api_key table v2" 2024-03-07T03:07:01.951 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.73734737Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-03-07T03:07:01.951 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.762505315Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-03-07T03:07:01.951 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.787782815Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-03-07T03:07:01.951 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.829669144Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-03-07T03:07:01.951 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.854943098Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-03-07T03:07:01.951 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.880164059Z level=info msg="Executing migration" id="Update api_key table charset" 2024-03-07T03:07:01.951 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.905358884Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-03-07T03:07:02.228 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.955699994Z level=info msg="Executing migration" id="Add service account foreign key" 2024-03-07T03:07:02.228 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.972642558Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-03-07T03:07:02.228 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:01 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:01.989528257Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-03-07T03:07:02.228 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.006298127Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-03-07T03:07:02.228 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.031545922Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-03-07T03:07:02.228 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.056762516Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-03-07T03:07:02.228 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.098790711Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-03-07T03:07:02.228 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.124079554Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-03-07T03:07:02.228 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.166002588Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-03-07T03:07:02.228 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.19135745Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-03-07T03:07:02.228 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.2332821Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-03-07T03:07:02.505 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[23614]: cluster 2024-03-07T03:07:00.372570+0000 mgr.y (mgr.24302) 53 : cluster 0 pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:02.505 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.258528141Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-03-07T03:07:02.505 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.283787431Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-03-07T03:07:02.505 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.342422695Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-03-07T03:07:02.505 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.384474209Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-03-07T03:07:02.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.409873001Z level=info msg="Executing migration" id="create quota table v1" 2024-03-07T03:07:02.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.426723335Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-03-07T03:07:02.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.468696299Z level=info msg="Executing migration" id="Update quota table charset" 2024-03-07T03:07:02.506 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.510722086Z level=info msg="Executing migration" id="create plugin_setting table" 2024-03-07T03:07:02.766 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.536070654Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-03-07T03:07:02.766 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.603083085Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-03-07T03:07:02.766 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.6538197Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-03-07T03:07:02.766 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.679129969Z level=info msg="Executing migration" id="create session table" 2024-03-07T03:07:02.766 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.721301093Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-03-07T03:07:02.766 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.746638506Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-03-07T03:07:02.766 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.771511539Z level=info msg="Executing migration" id="create playlist table v2" 2024-03-07T03:07:02.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:02 smithi145 bash[21339]: cluster 2024-03-07T03:07:00.372570+0000 mgr.y (mgr.24302) 53 : cluster 0 pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:02.816 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:02 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:07:02] "GET /metrics HTTP/1.1" 200 28552 "" "Prometheus/2.43.0" 2024-03-07T03:07:02.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:02 smithi145 bash[30906]: cluster 2024-03-07T03:07:00.372570+0000 mgr.y (mgr.24302) 53 : cluster 0 pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:03.027 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.796775706Z level=info msg="Executing migration" id="create playlist item table v2" 2024-03-07T03:07:03.027 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.830330013Z level=info msg="Executing migration" id="Update playlist table charset" 2024-03-07T03:07:03.027 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.872499664Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-03-07T03:07:03.027 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.897596603Z level=info msg="Executing migration" id="drop preferences table v2" 2024-03-07T03:07:03.027 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.93977204Z level=info msg="Executing migration" id="drop preferences table v3" 2024-03-07T03:07:03.027 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:02 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:02.981892127Z level=info msg="Executing migration" id="create preferences table v3" 2024-03-07T03:07:03.028 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.007226397Z level=info msg="Executing migration" id="Update preferences table charset" 2024-03-07T03:07:03.279 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.03247824Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-03-07T03:07:03.280 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.057646426Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-03-07T03:07:03.280 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.082955367Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-03-07T03:07:03.280 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.124976246Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-03-07T03:07:03.280 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.150224471Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-03-07T03:07:03.280 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.175424297Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-03-07T03:07:03.280 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.217464862Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-03-07T03:07:03.280 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.242849749Z level=info msg="Executing migration" id="create alert table v1" 2024-03-07T03:07:03.280 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.284790566Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-03-07T03:07:03.565 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.351797628Z level=info msg="Executing migration" id="add index alert state" 2024-03-07T03:07:03.565 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.377019957Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-03-07T03:07:03.565 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.40236839Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-03-07T03:07:03.566 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.444262038Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-03-07T03:07:03.566 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.486458055Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-03-07T03:07:03.566 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.511633811Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-03-07T03:07:03.566 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.570455324Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-03-07T03:07:03.566 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[23614]: cluster 2024-03-07T03:07:02.372975+0000 mgr.y (mgr.24302) 54 : cluster 0 pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:03.566 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[23614]: audit 2024-03-07T03:07:02.444195+0000 mon.a (mon.0) 732 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.566 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[23614]: audit 2024-03-07T03:07:02.445234+0000 mon.c (mon.2) 50 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:07:03.566 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[23614]: audit 2024-03-07T03:07:02.844592+0000 mon.a (mon.0) 733 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.566 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[23614]: audit 2024-03-07T03:07:02.851892+0000 mon.a (mon.0) 734 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.566 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[23614]: audit 2024-03-07T03:07:02.923857+0000 mon.a (mon.0) 735 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.566 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[23614]: audit 2024-03-07T03:07:02.933979+0000 mon.a (mon.0) 736 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.567 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[23614]: audit 2024-03-07T03:07:02.936500+0000 mon.c (mon.2) 51 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:03.567 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[23614]: audit 2024-03-07T03:07:02.937356+0000 mon.c (mon.2) 52 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:03.567 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[23614]: audit 2024-03-07T03:07:02.942744+0000 mon.a (mon.0) 737 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.567 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[23614]: cephadm 2024-03-07T03:07:03.012411+0000 mgr.y (mgr.24302) 55 : cephadm 1 Reconfiguring alertmanager.a (dependencies changed)... 2024-03-07T03:07:03.567 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[23614]: cephadm 2024-03-07T03:07:03.021216+0000 mgr.y (mgr.24302) 56 : cephadm 1 Reconfiguring daemon alertmanager.a on smithi145 2024-03-07T03:07:03.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[21339]: cluster 2024-03-07T03:07:02.372975+0000 mgr.y (mgr.24302) 54 : cluster 0 pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:03.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[21339]: audit 2024-03-07T03:07:02.444195+0000 mon.a (mon.0) 732 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[21339]: audit 2024-03-07T03:07:02.445234+0000 mon.c (mon.2) 50 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:07:03.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[21339]: audit 2024-03-07T03:07:02.844592+0000 mon.a (mon.0) 733 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[21339]: audit 2024-03-07T03:07:02.851892+0000 mon.a (mon.0) 734 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[21339]: audit 2024-03-07T03:07:02.923857+0000 mon.a (mon.0) 735 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[21339]: audit 2024-03-07T03:07:02.933979+0000 mon.a (mon.0) 736 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[21339]: audit 2024-03-07T03:07:02.936500+0000 mon.c (mon.2) 51 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:03.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[21339]: audit 2024-03-07T03:07:02.937356+0000 mon.c (mon.2) 52 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:03.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[21339]: audit 2024-03-07T03:07:02.942744+0000 mon.a (mon.0) 737 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[21339]: cephadm 2024-03-07T03:07:03.012411+0000 mgr.y (mgr.24302) 55 : cephadm 1 Reconfiguring alertmanager.a (dependencies changed)... 2024-03-07T03:07:03.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[21339]: cephadm 2024-03-07T03:07:03.021216+0000 mgr.y (mgr.24302) 56 : cephadm 1 Reconfiguring daemon alertmanager.a on smithi145 2024-03-07T03:07:03.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[30906]: cluster 2024-03-07T03:07:02.372975+0000 mgr.y (mgr.24302) 54 : cluster 0 pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:03.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[30906]: audit 2024-03-07T03:07:02.444195+0000 mon.a (mon.0) 732 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[30906]: audit 2024-03-07T03:07:02.445234+0000 mon.c (mon.2) 50 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:07:03.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[30906]: audit 2024-03-07T03:07:02.844592+0000 mon.a (mon.0) 733 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[30906]: audit 2024-03-07T03:07:02.851892+0000 mon.a (mon.0) 734 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[30906]: audit 2024-03-07T03:07:02.923857+0000 mon.a (mon.0) 735 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[30906]: audit 2024-03-07T03:07:02.933979+0000 mon.a (mon.0) 736 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[30906]: audit 2024-03-07T03:07:02.936500+0000 mon.c (mon.2) 51 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:03.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[30906]: audit 2024-03-07T03:07:02.937356+0000 mon.c (mon.2) 52 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:03.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[30906]: audit 2024-03-07T03:07:02.942744+0000 mon.a (mon.0) 737 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:03.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[30906]: cephadm 2024-03-07T03:07:03.012411+0000 mgr.y (mgr.24302) 55 : cephadm 1 Reconfiguring alertmanager.a (dependencies changed)... 2024-03-07T03:07:03.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:03 smithi145 bash[30906]: cephadm 2024-03-07T03:07:03.021216+0000 mgr.y (mgr.24302) 56 : cephadm 1 Reconfiguring daemon alertmanager.a on smithi145 2024-03-07T03:07:03.825 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.595785224Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-03-07T03:07:03.826 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.621061449Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-03-07T03:07:03.826 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.662863575Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-03-07T03:07:03.826 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.688219872Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-03-07T03:07:03.826 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.730114367Z level=info msg="Executing migration" id="Add column is_default" 2024-03-07T03:07:03.826 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.755331942Z level=info msg="Executing migration" id="Add column frequency" 2024-03-07T03:07:03.826 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.797320195Z level=info msg="Executing migration" id="Add column send_reminder" 2024-03-07T03:07:03.826 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.830881526Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-03-07T03:07:04.087 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.864575986Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-03-07T03:07:04.087 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.881453083Z level=info msg="Executing migration" id="Update alert table charset" 2024-03-07T03:07:04.087 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.898338475Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-03-07T03:07:04.087 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.915277127Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-03-07T03:07:04.087 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.93210977Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-03-07T03:07:04.087 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.957405057Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-03-07T03:07:04.087 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:03 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:03.990931293Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-03-07T03:07:04.087 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.016207108Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-03-07T03:07:04.087 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.041395045Z level=info msg="Executing migration" id="Add for to alert table" 2024-03-07T03:07:04.087 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.058197689Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-03-07T03:07:04.087 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.075117194Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-03-07T03:07:04.087 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.092071089Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-03-07T03:07:04.348 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.108998902Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-03-07T03:07:04.348 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.150942053Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-03-07T03:07:04.348 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.192908794Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-03-07T03:07:04.348 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.218148134Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-03-07T03:07:04.348 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.243357123Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-03-07T03:07:04.348 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.285358707Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-03-07T03:07:04.348 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.327351208Z level=info msg="Executing migration" id="create annotation table v5" 2024-03-07T03:07:04.348 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.352633496Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-03-07T03:07:04.608 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.377726068Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-03-07T03:07:04.608 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.403059772Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-03-07T03:07:04.608 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.428262672Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-03-07T03:07:04.608 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.470307828Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-03-07T03:07:04.608 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.495741161Z level=info msg="Executing migration" id="Update annotation table charset" 2024-03-07T03:07:04.609 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.520970417Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-03-07T03:07:04.609 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.546146227Z level=info msg="Executing migration" id="Drop category_id index" 2024-03-07T03:07:04.609 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.588103528Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-03-07T03:07:04.609 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.613381317Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-03-07T03:07:04.860 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.655419063Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-03-07T03:07:04.860 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.697372884Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-03-07T03:07:04.860 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.722617403Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-03-07T03:07:04.860 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.756175919Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-03-07T03:07:04.860 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.798105247Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-03-07T03:07:04.861 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.840161154Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-03-07T03:07:04.861 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.865441209Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-03-07T03:07:05.146 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.890747827Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-03-07T03:07:05.146 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.915871548Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-03-07T03:07:05.146 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.957787217Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-03-07T03:07:05.146 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:04 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:04.983135104Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-03-07T03:07:05.146 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.008407556Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-03-07T03:07:05.146 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.033639499Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-03-07T03:07:05.147 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.058902571Z level=info msg="Executing migration" id="Add epoch_end column" 2024-03-07T03:07:05.147 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.084128148Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-03-07T03:07:05.147 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.109363567Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-03-07T03:07:05.147 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.151259691Z level=info msg="Executing migration" id="Move region to single row" 2024-03-07T03:07:05.407 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.193247086Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-03-07T03:07:05.407 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.235107566Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-03-07T03:07:05.407 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.260441124Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-03-07T03:07:05.407 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.285624142Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-03-07T03:07:05.407 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.319278034Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-03-07T03:07:05.407 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.336205309Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-03-07T03:07:05.408 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.353122793Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-03-07T03:07:05.408 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.369988746Z level=info msg="Executing migration" id="create test_data table" 2024-03-07T03:07:05.408 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.387000817Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-03-07T03:07:05.408 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.412064563Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-03-07T03:07:05.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:05 smithi145 bash[21339]: cluster 2024-03-07T03:07:04.373724+0000 mgr.y (mgr.24302) 57 : cluster 0 pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:05.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:05 smithi145 bash[30906]: cluster 2024-03-07T03:07:04.373724+0000 mgr.y (mgr.24302) 57 : cluster 0 pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:05.676 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[23614]: cluster 2024-03-07T03:07:04.373724+0000 mgr.y (mgr.24302) 57 : cluster 0 pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:05.676 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.445930955Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-03-07T03:07:05.676 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.488095567Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-03-07T03:07:05.677 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.513347036Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-03-07T03:07:05.677 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.538610426Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-03-07T03:07:05.677 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.580439483Z level=info msg="Executing migration" id="create team table" 2024-03-07T03:07:05.677 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.605700663Z level=info msg="Executing migration" id="add index team.org_id" 2024-03-07T03:07:05.677 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.631117157Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-03-07T03:07:05.677 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.656317014Z level=info msg="Executing migration" id="create team member table" 2024-03-07T03:07:05.677 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.681548236Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-03-07T03:07:05.764 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:05.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.723460706Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-03-07T03:07:05.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.748731908Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-03-07T03:07:05.928 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.773910846Z level=info msg="Executing migration" id="Add column email to team table" 2024-03-07T03:07:05.929 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.815858558Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-03-07T03:07:05.929 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.857808533Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-03-07T03:07:05.929 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.883144851Z level=info msg="Executing migration" id="create dashboard acl table" 2024-03-07T03:07:05.929 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.908363725Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-03-07T03:07:05.929 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.93364015Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-03-07T03:07:06.180 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.958790828Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-03-07T03:07:06.180 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:05 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:05.984017608Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-03-07T03:07:06.180 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.025933398Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-03-07T03:07:06.180 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.051139269Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-03-07T03:07:06.180 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.076347879Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-03-07T03:07:06.180 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.1182456Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-03-07T03:07:06.180 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.160226395Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-03-07T03:07:06.180 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.185455513Z level=info msg="Executing migration" id="create tag table" 2024-03-07T03:07:06.441 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.210722806Z level=info msg="Executing migration" id="add index tag.key_value" 2024-03-07T03:07:06.442 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.235968293Z level=info msg="Executing migration" id="create login attempt table" 2024-03-07T03:07:06.442 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.277881413Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-03-07T03:07:06.442 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.294919471Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-03-07T03:07:06.442 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.311773566Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-03-07T03:07:06.442 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.337030342Z level=info msg="Executing migration" id="create login_attempt v2" 2024-03-07T03:07:06.442 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.362617067Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-03-07T03:07:06.442 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.387855757Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-03-07T03:07:06.442 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.413019635Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-03-07T03:07:06.442 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.429811347Z level=info msg="Executing migration" id="create user auth table" 2024-03-07T03:07:06.442 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.446798876Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-03-07T03:07:06.694 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.480457975Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-03-07T03:07:06.694 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.497304279Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-03-07T03:07:06.694 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.514203536Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-03-07T03:07:06.695 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.531111146Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-03-07T03:07:06.695 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.548009446Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-03-07T03:07:06.695 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.581572077Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-03-07T03:07:06.695 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.598533203Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-03-07T03:07:06.695 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.615360134Z level=info msg="Executing migration" id="create server_lock table" 2024-03-07T03:07:06.695 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.632344876Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-03-07T03:07:06.695 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.665916923Z level=info msg="Executing migration" id="create user auth token table" 2024-03-07T03:07:06.695 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.699605242Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-03-07T03:07:06.955 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.716575704Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-03-07T03:07:06.955 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.741757925Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-03-07T03:07:06.955 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.76711376Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-03-07T03:07:06.955 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.792282995Z level=info msg="Executing migration" id="create cache_data table" 2024-03-07T03:07:06.955 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.83433662Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-03-07T03:07:06.955 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.85958012Z level=info msg="Executing migration" id="create short_url table v1" 2024-03-07T03:07:06.955 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.88481667Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-03-07T03:07:06.955 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:06 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:06.960031138Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-03-07T03:07:07.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.002003567Z level=info msg="Executing migration" id="delete alert_definition table" 2024-03-07T03:07:07.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.027289374Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-03-07T03:07:07.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.052565535Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-03-07T03:07:07.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.077672917Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-03-07T03:07:07.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.102967087Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-03-07T03:07:07.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.144920511Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-03-07T03:07:07.208 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.170257547Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-03-07T03:07:07.208 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.212172722Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-03-07T03:07:07.468 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[23614]: cluster 2024-03-07T03:07:06.374361+0000 mgr.y (mgr.24302) 58 : cluster 0 pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:07.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.237637976Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-03-07T03:07:07.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.263000381Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-03-07T03:07:07.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.304908869Z level=info msg="Executing migration" id="drop alert_definition table" 2024-03-07T03:07:07.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.330183516Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-03-07T03:07:07.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.355439764Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-03-07T03:07:07.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.380681991Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-03-07T03:07:07.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.431009389Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-03-07T03:07:07.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.4731109Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-03-07T03:07:07.720 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.498380844Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-03-07T03:07:07.721 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.515298818Z level=info msg="Executing migration" id="create alert_instance table" 2024-03-07T03:07:07.721 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.532209791Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-03-07T03:07:07.721 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.549139085Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-03-07T03:07:07.721 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.566043486Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-03-07T03:07:07.721 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.591265853Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-03-07T03:07:07.721 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.624896026Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-03-07T03:07:07.721 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.641739408Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-03-07T03:07:07.721 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.675411544Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-03-07T03:07:07.721 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.725669759Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-03-07T03:07:07.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:07 smithi145 bash[21339]: cluster 2024-03-07T03:07:06.374361+0000 mgr.y (mgr.24302) 58 : cluster 0 pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:07.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:07 smithi145 bash[30906]: cluster 2024-03-07T03:07:06.374361+0000 mgr.y (mgr.24302) 58 : cluster 0 pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:08.022 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.751049841Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-03-07T03:07:08.022 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.776239266Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-03-07T03:07:08.023 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.801558745Z level=info msg="Executing migration" id="create alert_rule table" 2024-03-07T03:07:08.023 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.851888636Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-03-07T03:07:08.023 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.877102884Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-03-07T03:07:08.023 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:07 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:07.902405987Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-03-07T03:07:08.301 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.027750816Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-03-07T03:07:08.302 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.086257141Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-03-07T03:07:08.302 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.186578885Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-03-07T03:07:08.584 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.307128082Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-03-07T03:07:08.585 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.353972185Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-03-07T03:07:08.585 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.379584427Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-03-07T03:07:08.585 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.404767829Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-03-07T03:07:08.585 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.446732637Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-03-07T03:07:08.585 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.472085327Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-03-07T03:07:08.585 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.497410593Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-03-07T03:07:08.585 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.522605636Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-03-07T03:07:08.585 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.547849254Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-03-07T03:07:08.585 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.589847785Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-03-07T03:07:08.755 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:08 smithi145 systemd[1]: Stopping Ceph alertmanager.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:07:08.755 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:08 smithi145 bash[56172]: ts=2024-03-07T03:07:08.564Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-03-07T03:07:08.836 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.615148328Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-03-07T03:07:08.836 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.657159719Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-03-07T03:07:08.836 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.699144454Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-03-07T03:07:08.836 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.716011865Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-03-07T03:07:08.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.732970938Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-03-07T03:07:08.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.758210293Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-03-07T03:07:08.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.775020848Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-03-07T03:07:08.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.792014461Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-03-07T03:07:08.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.833942Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-03-07T03:07:09.123 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.86762711Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-03-07T03:07:09.123 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.892938957Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-03-07T03:07:09.123 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.918110216Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-03-07T03:07:09.124 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.94335124Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-03-07T03:07:09.124 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:08 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:08.985462957Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-03-07T03:07:09.124 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.027390562Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-03-07T03:07:09.124 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.052607965Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-03-07T03:07:09.124 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.077782268Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-03-07T03:07:09.124 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.103135858Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-03-07T03:07:09.124 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.1283791Z level=info msg="Executing migration" id="create provenance_type table" 2024-03-07T03:07:09.392 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.170406771Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-03-07T03:07:09.392 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.212256495Z level=info msg="Executing migration" id="create alert_image table" 2024-03-07T03:07:09.392 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.237571716Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-03-07T03:07:09.392 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.263033848Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-03-07T03:07:09.392 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.288258139Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-03-07T03:07:09.393 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.330148646Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-03-07T03:07:09.393 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.37227143Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-03-07T03:07:09.393 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.372642421Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-03-07T03:07:09.393 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.397563584Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-03-07T03:07:09.653 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.422718298Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-03-07T03:07:09.653 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.447987056Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-03-07T03:07:09.653 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.448531861Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-03-07T03:07:09.653 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.448942692Z level=info msg="alerts found to migrate" alerts=0 2024-03-07T03:07:09.653 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.473262594Z level=info msg="Executing migration" id="create library_element table v1" 2024-03-07T03:07:09.653 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.498520572Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-03-07T03:07:09.653 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.523779673Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-03-07T03:07:09.653 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.548981189Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-03-07T03:07:09.654 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.574236254Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-03-07T03:07:09.654 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.616212266Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-03-07T03:07:09.654 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.658082937Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-03-07T03:07:09.840 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:09 smithi145 bash[57800]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-alertmanager-a 2024-03-07T03:07:09.938 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.683375414Z level=info msg="Executing migration" id="create data_keys table" 2024-03-07T03:07:09.938 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.725373083Z level=info msg="Executing migration" id="create secrets table" 2024-03-07T03:07:09.938 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.750629151Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-03-07T03:07:09.938 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.792459322Z level=info msg="Executing migration" id="add name column into data_keys" 2024-03-07T03:07:09.938 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.834401872Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-03-07T03:07:09.938 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.859700349Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-03-07T03:07:09.938 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.901570603Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-03-07T03:07:09.939 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.943449564Z level=info msg="Executing migration" id="create kv_store table v1" 2024-03-07T03:07:10.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.968708105Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-03-07T03:07:10.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:09 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:09.993917191Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-03-07T03:07:10.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.019079847Z level=info msg="Executing migration" id="create permission table" 2024-03-07T03:07:10.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.061166902Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-03-07T03:07:10.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.086340799Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-03-07T03:07:10.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.128292656Z level=info msg="Executing migration" id="create role table" 2024-03-07T03:07:10.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.170297059Z level=info msg="Executing migration" id="add column display_name" 2024-03-07T03:07:10.207 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.212209612Z level=info msg="Executing migration" id="add column group_name" 2024-03-07T03:07:10.240 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:09 smithi145 bash[57873]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-alertmanager-a 2024-03-07T03:07:10.240 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:09 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-03-07T03:07:10.240 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:09 smithi145 systemd[1]: Stopped Ceph alertmanager.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:07:10.240 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:09 smithi145 systemd[1]: Started Ceph alertmanager.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:07:10.467 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[23614]: cluster 2024-03-07T03:07:08.374872+0000 mgr.y (mgr.24302) 59 : cluster 0 pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:10.467 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[23614]: audit 2024-03-07T03:07:10.024895+0000 mon.a (mon.0) 738 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:10.468 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[23614]: audit 2024-03-07T03:07:10.033133+0000 mon.a (mon.0) 739 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:10.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.254151Z level=info msg="Executing migration" id="add index role.org_id" 2024-03-07T03:07:10.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.279609624Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-03-07T03:07:10.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.321563898Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-03-07T03:07:10.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.363801198Z level=info msg="Executing migration" id="create team role table" 2024-03-07T03:07:10.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.405806768Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-03-07T03:07:10.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.447790821Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-03-07T03:07:10.468 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.472995723Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-03-07T03:07:10.540 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:10 smithi145 bash[21339]: cluster 2024-03-07T03:07:08.374872+0000 mgr.y (mgr.24302) 59 : cluster 0 pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:10.540 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:10 smithi145 bash[21339]: audit 2024-03-07T03:07:10.024895+0000 mon.a (mon.0) 738 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:10.540 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:10 smithi145 bash[21339]: audit 2024-03-07T03:07:10.033133+0000 mon.a (mon.0) 739 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:10.540 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:10 smithi145 bash[30906]: cluster 2024-03-07T03:07:08.374872+0000 mgr.y (mgr.24302) 59 : cluster 0 pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:10.540 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:10 smithi145 bash[30906]: audit 2024-03-07T03:07:10.024895+0000 mon.a (mon.0) 738 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:10.540 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:10 smithi145 bash[30906]: audit 2024-03-07T03:07:10.033133+0000 mon.a (mon.0) 739 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:10.620 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:07:10.620 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":64,"fsid":"8947a0f8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:55:42.039360+0000","modified":"2024-03-07T03:06:51.319603+0000","last_up_change":"2024-03-07T03:05:04.610834+0000","last_in_change":"2024-03-07T03:04:39.932769+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:00:59.846304+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-07T03:06:41.584515+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"242c63ee-0417-4ebf-bd71-6239f5d3634f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6803","nonce":1367316946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6805","nonce":1367316946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6809","nonce":1367316946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6807","nonce":1367316946}]},"public_addr":"172.21.15.145:6803/1367316946","cluster_addr":"172.21.15.145:6805/1367316946","heartbeat_back_addr":"172.21.15.145:6809/1367316946","heartbeat_front_addr":"172.21.15.145:6807/1367316946","state":["exists","up"]},{"osd":1,"uuid":"3c2818da-5d32-4f7a-8a63-8d7d679961e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6811","nonce":1873038359}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6813","nonce":1873038359}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6817","nonce":1873038359}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6815","nonce":1873038359}]},"public_addr":"172.21.15.145:6811/1873038359","cluster_addr":"172.21.15.145:6813/1873038359","heartbeat_back_addr":"172.21.15.145:6817/1873038359","heartbeat_front_addr":"172.21.15.145:6815/1873038359","state":["exists","up"]},{"osd":2,"uuid":"905ec219-dd9c-4f30-bdab-ed78d1405c11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6819","nonce":462216015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6821","nonce":462216015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6825","nonce":462216015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6823","nonce":462216015}]},"public_addr":"172.21.15.145:6819/462216015","cluster_addr":"172.21.15.145:6821/462216015","heartbeat_back_addr":"172.21.15.145:6825/462216015","heartbeat_front_addr":"172.21.15.145:6823/462216015","state":["exists","up"]},{"osd":3,"uuid":"5f466751-f5a7-4d18-87ab-96251b77b663","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6827","nonce":2306759084}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6829","nonce":2306759084}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6833","nonce":2306759084}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6831","nonce":2306759084}]},"public_addr":"172.21.15.145:6827/2306759084","cluster_addr":"172.21.15.145:6829/2306759084","heartbeat_back_addr":"172.21.15.145:6833/2306759084","heartbeat_front_addr":"172.21.15.145:6831/2306759084","state":["exists","up"]},{"osd":4,"uuid":"633f56af-3ff8-4954-97b4-bc8122bb10ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6801","nonce":3152992669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6803","nonce":3152992669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6807","nonce":3152992669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6805","nonce":3152992669}]},"public_addr":"172.21.15.189:6801/3152992669","cluster_addr":"172.21.15.189:6803/3152992669","heartbeat_back_addr":"172.21.15.189:6807/3152992669","heartbeat_front_addr":"172.21.15.189:6805/3152992669","state":["exists","up"]},{"osd":5,"uuid":"531d5127-60fe-4669-ab7d-fe4f5e5753cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6809","nonce":2096444201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6811","nonce":2096444201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6815","nonce":2096444201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6813","nonce":2096444201}]},"public_addr":"172.21.15.189:6809/2096444201","cluster_addr":"172.21.15.189:6811/2096444201","heartbeat_back_addr":"172.21.15.189:6815/2096444201","heartbeat_front_addr":"172.21.15.189:6813/2096444201","state":["exists","up"]},{"osd":6,"uuid":"8f2b456e-712e-45f1-8d84-0fdc4a6d860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6817","nonce":4219543665}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6819","nonce":4219543665}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6823","nonce":4219543665}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6821","nonce":4219543665}]},"public_addr":"172.21.15.189:6817/4219543665","cluster_addr":"172.21.15.189:6819/4219543665","heartbeat_back_addr":"172.21.15.189:6823/4219543665","heartbeat_front_addr":"172.21.15.189:6821/4219543665","state":["exists","up"]},{"osd":7,"uuid":"24dc7eba-4cd4-414d-93d2-598f5f2e655c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6825","nonce":452348965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6827","nonce":452348965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6831","nonce":452348965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6829","nonce":452348965}]},"public_addr":"172.21.15.189:6825/452348965","cluster_addr":"172.21.15.189:6827/452348965","heartbeat_back_addr":"172.21.15.189:6831/452348965","heartbeat_front_addr":"172.21.15.189:6829/452348965","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T02:59:11.112897+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:03.501748+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:56.635004+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:01:50.372735+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:38.150721+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:25.965239+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:14.345171+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:03.479901+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.145:6801/2891889777":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3299433409":"2024-03-08T03:06:02.331013+0000","172.21.15.145:6800/2891889777":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/2939737297":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/3852118874":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3488231986":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/2236140381":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/1978862755":"2024-03-08T03:06:02.331013+0000","172.21.15.145:6800/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/2696605014":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/131277059":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/2414597158":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:6800/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/321164949":"2024-03-08T02:56:14.659813+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:07:10.785 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.531538686Z level=info msg="Executing migration" id="create user role table" 2024-03-07T03:07:10.785 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.573216655Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-03-07T03:07:11.053 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:10 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:10.79078386Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-03-07T03:07:11.328 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:11 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:11.05844524Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-03-07T03:07:11.328 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:11 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:11.178308779Z level=info msg="Executing migration" id="create builtin role table" 2024-03-07T03:07:11.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:11 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:11.435412142Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-03-07T03:07:11.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:11 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:11.544114241Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-03-07T03:07:11.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:11 smithi189 bash[23614]: cephadm 2024-03-07T03:07:10.052343+0000 mgr.y (mgr.24302) 60 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:07:11.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:11 smithi189 bash[23614]: cephadm 2024-03-07T03:07:10.284095+0000 mgr.y (mgr.24302) 61 : cephadm 1 Reconfiguring daemon prometheus.a on smithi189 2024-03-07T03:07:11.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:11 smithi189 bash[23614]: audit 2024-03-07T03:07:10.623558+0000 mon.a (mon.0) 740 : audit 0 from='client.? 172.21.15.145:0/2807215135' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:07:11.613 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:11 smithi145 bash[21339]: cephadm 2024-03-07T03:07:10.052343+0000 mgr.y (mgr.24302) 60 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:07:11.613 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:11 smithi145 bash[21339]: cephadm 2024-03-07T03:07:10.284095+0000 mgr.y (mgr.24302) 61 : cephadm 1 Reconfiguring daemon prometheus.a on smithi189 2024-03-07T03:07:11.613 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:11 smithi145 bash[21339]: audit 2024-03-07T03:07:10.623558+0000 mon.a (mon.0) 740 : audit 0 from='client.? 172.21.15.145:0/2807215135' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:07:11.613 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:11 smithi145 bash[30906]: cephadm 2024-03-07T03:07:10.052343+0000 mgr.y (mgr.24302) 60 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:07:11.614 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:11 smithi145 bash[30906]: cephadm 2024-03-07T03:07:10.284095+0000 mgr.y (mgr.24302) 61 : cephadm 1 Reconfiguring daemon prometheus.a on smithi189 2024-03-07T03:07:11.614 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:11 smithi145 bash[30906]: audit 2024-03-07T03:07:10.623558+0000 mon.a (mon.0) 740 : audit 0 from='client.? 172.21.15.145:0/2807215135' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:07:11.916 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:11 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:11.62187764Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-03-07T03:07:11.916 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:11 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:11.695343662Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-03-07T03:07:11.917 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:11 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:11.772015213Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-03-07T03:07:11.917 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:11 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:11.854829648Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-03-07T03:07:12.025 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-07T03:07:12.025 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:07:12.204 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:11 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:11.921979069Z level=info msg="Executing migration" id="add unique index role.uid" 2024-03-07T03:07:12.204 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:12 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:12.000391592Z level=info msg="Executing migration" id="create seed assignment table" 2024-03-07T03:07:12.204 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:12 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:12.0560669Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-03-07T03:07:12.204 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:12 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:12.098386206Z level=info msg="Executing migration" id="add column hidden to role table" 2024-03-07T03:07:12.204 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:12 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:12.148863063Z level=info msg="Executing migration" id="create query_history table v1" 2024-03-07T03:07:12.316 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:12 smithi145 bash[57902]: ts=2024-03-07T03:07:12.150Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-03-07T03:07:12.316 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:12 smithi145 bash[57902]: ts=2024-03-07T03:07:12.150Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-03-07T03:07:12.317 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:12 smithi145 bash[57902]: ts=2024-03-07T03:07:12.151Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.145 port=9094 2024-03-07T03:07:12.317 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:12 smithi145 bash[57902]: ts=2024-03-07T03:07:12.152Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-03-07T03:07:12.317 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:12 smithi145 bash[57902]: ts=2024-03-07T03:07:12.187Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-07T03:07:12.317 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:12 smithi145 bash[57902]: ts=2024-03-07T03:07:12.188Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-07T03:07:12.317 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:12 smithi145 bash[57902]: ts=2024-03-07T03:07:12.191Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-03-07T03:07:12.317 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:12 smithi145 bash[57902]: ts=2024-03-07T03:07:12.191Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-03-07T03:07:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:12 smithi189 bash[23614]: cluster 2024-03-07T03:07:10.375279+0000 mgr.y (mgr.24302) 62 : cluster 0 pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:12.637 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:12 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:12.216178414Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-03-07T03:07:12.637 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:12 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:12.266568482Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-03-07T03:07:12.637 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:12 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:12.350324932Z level=info msg="Executing migration" id="teams permissions migration" 2024-03-07T03:07:12.637 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:12 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:12.450700921Z level=info msg="Executing migration" id="dashboard permissions" 2024-03-07T03:07:12.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:12 smithi145 bash[21339]: cluster 2024-03-07T03:07:10.375279+0000 mgr.y (mgr.24302) 62 : cluster 0 pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:12.817 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:12 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:07:12] "GET /metrics HTTP/1.1" 200 28556 "" "Prometheus/2.43.0" 2024-03-07T03:07:12.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:12 smithi145 bash[30906]: cluster 2024-03-07T03:07:10.375279+0000 mgr.y (mgr.24302) 62 : cluster 0 pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:12.939 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:12 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:12.645694091Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-03-07T03:07:12.939 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:12 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:12.858765018Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-03-07T03:07:13.278 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:12 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:12.944254326Z level=info msg="Executing migration" id="alerting notification permissions" 2024-03-07T03:07:13.278 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:13 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:13.019998757Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-03-07T03:07:13.278 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:13 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:13.112671653Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-03-07T03:07:13.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:13 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:13.283540637Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-03-07T03:07:13.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:13 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:13.494045867Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-03-07T03:07:13.855 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:13 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:13.606742218Z level=info msg="Executing migration" id="create correlation table v1" 2024-03-07T03:07:14.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:13 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:13.860683643Z level=info msg="Executing migration" id="add index correlations.uid" 2024-03-07T03:07:14.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:13 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:13.966432404Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-03-07T03:07:14.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.038221463Z level=info msg="Executing migration" id="add correlation config column" 2024-03-07T03:07:14.137 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.142344262Z level=info msg="Executing migration" id="create entity_events table" 2024-03-07T03:07:14.398 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[23614]: cluster 2024-03-07T03:07:12.375783+0000 mgr.y (mgr.24302) 63 : cluster 0 pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:14.399 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.232306764Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-03-07T03:07:14.399 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.27662382Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-03-07T03:07:14.399 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.277013498Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-03-07T03:07:14.399 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.333453317Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-07T03:07:14.399 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.333814802Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-07T03:07:14.399 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.403978756Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-03-07T03:07:14.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:14 smithi145 bash[21339]: cluster 2024-03-07T03:07:12.375783+0000 mgr.y (mgr.24302) 63 : cluster 0 pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:14.566 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:14 smithi145 bash[57902]: ts=2024-03-07T03:07:14.153Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000758059s 2024-03-07T03:07:14.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:14 smithi145 bash[30906]: cluster 2024-03-07T03:07:12.375783+0000 mgr.y (mgr.24302) 63 : cluster 0 pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:14.649 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.444621427Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-03-07T03:07:14.649 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.48677685Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-03-07T03:07:14.649 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.52894245Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-07T03:07:14.649 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.613354617Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-03-07T03:07:14.649 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.655129338Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-03-07T03:07:14.926 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.740629345Z level=info msg="Executing migration" id="Drop public config table" 2024-03-07T03:07:14.926 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.829687053Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-03-07T03:07:15.330 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:14 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:14.931723787Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-03-07T03:07:15.331 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:15.144668752Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-03-07T03:07:15.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:15.33628818Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-03-07T03:07:15.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:15.483728209Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-03-07T03:07:15.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:15.559288616Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-03-07T03:07:15.960 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:15.618299612Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-03-07T03:07:15.960 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:15.660275493Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-03-07T03:07:15.960 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:15.702380972Z level=info msg="Executing migration" id="add share column" 2024-03-07T03:07:15.960 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:15.752903787Z level=info msg="Executing migration" id="create default alerting folders" 2024-03-07T03:07:15.960 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:15.78681779Z level=info msg="Executing migration" id="create file table" 2024-03-07T03:07:15.960 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:15.820484539Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-03-07T03:07:15.960 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:15.862771274Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-03-07T03:07:15.960 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:15.964939569Z level=info msg="Executing migration" id="create file_meta table" 2024-03-07T03:07:15.961 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:15 smithi189 systemd[1]: Stopping Ceph prometheus.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:07:15.961 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[47817]: ts=2024-03-07T03:07:15.760Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-07T03:07:15.961 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[47817]: ts=2024-03-07T03:07:15.760Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-07T03:07:15.961 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[47817]: ts=2024-03-07T03:07:15.760Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-07T03:07:15.961 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[47817]: ts=2024-03-07T03:07:15.760Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-07T03:07:15.961 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[47817]: ts=2024-03-07T03:07:15.760Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-07T03:07:15.961 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[47817]: ts=2024-03-07T03:07:15.760Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-07T03:07:15.961 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[47817]: ts=2024-03-07T03:07:15.761Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-07T03:07:15.961 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[47817]: ts=2024-03-07T03:07:15.761Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-07T03:07:15.961 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[47817]: ts=2024-03-07T03:07:15.761Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-07T03:07:15.961 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[47817]: ts=2024-03-07T03:07:15.761Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-07T03:07:15.961 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[47817]: ts=2024-03-07T03:07:15.761Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-07T03:07:15.961 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:15 smithi189 bash[47817]: ts=2024-03-07T03:07:15.761Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-07T03:07:16.242 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:16 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:16.056542213Z level=info msg="Executing migration" id="file table idx: path key" 2024-03-07T03:07:16.496 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:16 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:16.247927558Z level=info msg="Executing migration" id="set path collation in file table" 2024-03-07T03:07:16.496 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:16 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:16.47343613Z level=info msg="Executing migration" id="managed permissions migration" 2024-03-07T03:07:16.496 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:16 smithi189 bash[23614]: cluster 2024-03-07T03:07:14.376493+0000 mgr.y (mgr.24302) 64 : cluster 0 pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:16.763 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:16.785 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:16 smithi145 bash[21339]: cluster 2024-03-07T03:07:14.376493+0000 mgr.y (mgr.24302) 64 : cluster 0 pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:16.785 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:16 smithi145 bash[30906]: cluster 2024-03-07T03:07:14.376493+0000 mgr.y (mgr.24302) 64 : cluster 0 pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:16.786 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:16 smithi145 bash[21623]: [07/Mar/2024:03:07:16] ENGINE Bus STOPPING 2024-03-07T03:07:16.805 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:16 smithi189 bash[50169]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-prometheus-a 2024-03-07T03:07:16.805 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:16 smithi189 bash[50229]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-prometheus-a 2024-03-07T03:07:16.805 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:16 smithi189 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-07T03:07:16.805 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:16 smithi189 systemd[1]: Stopped Ceph prometheus.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:07:16.805 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:16 smithi189 systemd[1]: Started Ceph prometheus.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:07:16.805 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:16 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:16.565503882Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-03-07T03:07:16.805 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:16 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:16.641363634Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-03-07T03:07:16.805 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:16 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:16.709452029Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-03-07T03:07:17.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:16 smithi145 bash[21623]: [07/Mar/2024:03:07:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:07:17.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:16 smithi145 bash[21623]: [07/Mar/2024:03:07:16] ENGINE Bus STOPPED 2024-03-07T03:07:17.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:16 smithi145 bash[21623]: [07/Mar/2024:03:07:16] ENGINE Bus STARTING 2024-03-07T03:07:17.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:16 smithi145 bash[21623]: [07/Mar/2024:03:07:16] ENGINE Serving on http://:::9283 2024-03-07T03:07:17.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:16 smithi145 bash[21623]: [07/Mar/2024:03:07:16] ENGINE Bus STARTED 2024-03-07T03:07:17.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:16 smithi145 bash[21623]: [07/Mar/2024:03:07:16] ENGINE Bus STOPPING 2024-03-07T03:07:17.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:16 smithi145 bash[21623]: [07/Mar/2024:03:07:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:07:17.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:16 smithi145 bash[21623]: [07/Mar/2024:03:07:16] ENGINE Bus STOPPED 2024-03-07T03:07:17.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:16 smithi145 bash[21623]: [07/Mar/2024:03:07:16] ENGINE Bus STARTING 2024-03-07T03:07:17.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21623]: [07/Mar/2024:03:07:17] ENGINE Serving on http://:::9283 2024-03-07T03:07:17.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21623]: [07/Mar/2024:03:07:17] ENGINE Bus STARTED 2024-03-07T03:07:17.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21623]: [07/Mar/2024:03:07:17] ENGINE Bus STOPPING 2024-03-07T03:07:17.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21623]: [07/Mar/2024:03:07:17] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:07:17.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21623]: [07/Mar/2024:03:07:17] ENGINE Bus STOPPED 2024-03-07T03:07:17.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21623]: [07/Mar/2024:03:07:17] ENGINE Bus STARTING 2024-03-07T03:07:17.086 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:16 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:16.810685131Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-03-07T03:07:17.086 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:17.056133016Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-03-07T03:07:17.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21623]: [07/Mar/2024:03:07:17] ENGINE Serving on http://:::9283 2024-03-07T03:07:17.566 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21623]: [07/Mar/2024:03:07:17] ENGINE Bus STARTED 2024-03-07T03:07:17.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:17.320368059Z level=info msg="Executing migration" id="update group index for alert rules" 2024-03-07T03:07:17.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:17.474645762Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-03-07T03:07:17.893 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:17.61361794Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-03-07T03:07:17.893 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:17.747813449Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-03-07T03:07:17.893 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:17.831560383Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-03-07T03:07:17.893 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: cluster 2024-03-07T03:07:16.377091+0000 mgr.y (mgr.24302) 65 : cluster 0 pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:17.894 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.738689+0000 mon.a (mon.0) 741 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:17.894 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.755470+0000 mon.a (mon.0) 742 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:17.894 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.773428+0000 mon.c (mon.2) 53 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-07T03:07:17.894 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.774092+0000 mgr.y (mgr.24302) 66 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-07T03:07:17.894 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.775589+0000 mon.c (mon.2) 54 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145.front.sepia.ceph.com:9093"}]: dispatch 2024-03-07T03:07:17.894 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.776141+0000 mgr.y (mgr.24302) 67 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145.front.sepia.ceph.com:9093"}]: dispatch 2024-03-07T03:07:17.894 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.784288+0000 mon.a (mon.0) 743 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:17.894 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.796754+0000 mon.c (mon.2) 55 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:07:17.894 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.797300+0000 mgr.y (mgr.24302) 68 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:07:17.894 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.799427+0000 mon.c (mon.2) 56 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi189.front.sepia.ceph.com:9095"}]: dispatch 2024-03-07T03:07:17.894 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.799856+0000 mgr.y (mgr.24302) 69 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi189.front.sepia.ceph.com:9095"}]: dispatch 2024-03-07T03:07:17.895 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.808930+0000 mon.a (mon.0) 744 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:17.895 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.818667+0000 mon.c (mon.2) 57 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-07T03:07:17.895 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.819185+0000 mgr.y (mgr.24302) 70 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-07T03:07:17.895 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.821207+0000 mon.c (mon.2) 58 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi189.front.sepia.ceph.com:3000"}]: dispatch 2024-03-07T03:07:17.895 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.821590+0000 mgr.y (mgr.24302) 71 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi189.front.sepia.ceph.com:3000"}]: dispatch 2024-03-07T03:07:17.895 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.829473+0000 mon.a (mon.0) 745 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:17.895 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:16.843844+0000 mon.c (mon.2) 59 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:17.895 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[23614]: audit 2024-03-07T03:07:17.441733+0000 mon.c (mon.2) 60 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:07:18.006 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: cluster 2024-03-07T03:07:16.377091+0000 mgr.y (mgr.24302) 65 : cluster 0 pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.738689+0000 mon.a (mon.0) 741 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.755470+0000 mon.a (mon.0) 742 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.773428+0000 mon.c (mon.2) 53 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.774092+0000 mgr.y (mgr.24302) 66 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.775589+0000 mon.c (mon.2) 54 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145.front.sepia.ceph.com:9093"}]: dispatch 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.776141+0000 mgr.y (mgr.24302) 67 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145.front.sepia.ceph.com:9093"}]: dispatch 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.784288+0000 mon.a (mon.0) 743 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.796754+0000 mon.c (mon.2) 55 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.797300+0000 mgr.y (mgr.24302) 68 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.799427+0000 mon.c (mon.2) 56 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi189.front.sepia.ceph.com:9095"}]: dispatch 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.799856+0000 mgr.y (mgr.24302) 69 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi189.front.sepia.ceph.com:9095"}]: dispatch 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.808930+0000 mon.a (mon.0) 744 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.818667+0000 mon.c (mon.2) 57 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-07T03:07:18.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.819185+0000 mgr.y (mgr.24302) 70 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-07T03:07:18.008 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.821207+0000 mon.c (mon.2) 58 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi189.front.sepia.ceph.com:3000"}]: dispatch 2024-03-07T03:07:18.008 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.821590+0000 mgr.y (mgr.24302) 71 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi189.front.sepia.ceph.com:3000"}]: dispatch 2024-03-07T03:07:18.008 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.829473+0000 mon.a (mon.0) 745 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:18.008 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:16.843844+0000 mon.c (mon.2) 59 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:18.008 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[21339]: audit 2024-03-07T03:07:17.441733+0000 mon.c (mon.2) 60 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:07:18.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: cluster 2024-03-07T03:07:16.377091+0000 mgr.y (mgr.24302) 65 : cluster 0 pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:18.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.738689+0000 mon.a (mon.0) 741 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:18.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.755470+0000 mon.a (mon.0) 742 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:18.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.773428+0000 mon.c (mon.2) 53 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-07T03:07:18.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.774092+0000 mgr.y (mgr.24302) 66 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.775589+0000 mon.c (mon.2) 54 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145.front.sepia.ceph.com:9093"}]: dispatch 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.776141+0000 mgr.y (mgr.24302) 67 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi145.front.sepia.ceph.com:9093"}]: dispatch 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.784288+0000 mon.a (mon.0) 743 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.796754+0000 mon.c (mon.2) 55 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.797300+0000 mgr.y (mgr.24302) 68 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.799427+0000 mon.c (mon.2) 56 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi189.front.sepia.ceph.com:9095"}]: dispatch 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.799856+0000 mgr.y (mgr.24302) 69 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi189.front.sepia.ceph.com:9095"}]: dispatch 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.808930+0000 mon.a (mon.0) 744 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.818667+0000 mon.c (mon.2) 57 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.819185+0000 mgr.y (mgr.24302) 70 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.821207+0000 mon.c (mon.2) 58 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi189.front.sepia.ceph.com:3000"}]: dispatch 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.821590+0000 mgr.y (mgr.24302) 71 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi189.front.sepia.ceph.com:3000"}]: dispatch 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.829473+0000 mon.a (mon.0) 745 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:16.843844+0000 mon.c (mon.2) 59 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:18.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:17 smithi145 bash[30906]: audit 2024-03-07T03:07:17.441733+0000 mon.c (mon.2) 60 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:07:18.148 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:17.898682411Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-03-07T03:07:18.148 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:17 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:17.974137373Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-03-07T03:07:18.148 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:18.086648098Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-03-07T03:07:18.149 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:18.133211023Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-03-07T03:07:18.440 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:18.175319965Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-03-07T03:07:18.440 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:18.226042877Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-03-07T03:07:18.440 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:18.293623625Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-03-07T03:07:18.440 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:18.31050492Z level=info msg="Executing migration" id="create folder table" 2024-03-07T03:07:18.440 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:18.344118162Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-03-07T03:07:18.440 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:18.361166537Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-03-07T03:07:18.440 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:18.378047086Z level=info msg="Executing migration" id="Update folder title length" 2024-03-07T03:07:18.440 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:18.403277199Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-03-07T03:07:18.440 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=migrator t=2024-03-07T03:07:18.445274896Z level=info msg="migrations completed" performed=485 skipped=0 duration=20.556992125s 2024-03-07T03:07:18.441 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.216Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-07T03:07:18.441 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.216Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-07T03:07:18.441 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.216Z caller=main.go:570 level=info host_details="(Linux 5.15.0-97-generic #107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024 x86_64 smithi189 (none))" 2024-03-07T03:07:18.441 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.216Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-07T03:07:18.441 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.216Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-07T03:07:18.441 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.218Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-07T03:07:18.441 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.218Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-07T03:07:18.441 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.219Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-07T03:07:18.441 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.219Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-07T03:07:18.441 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.222Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-07T03:07:18.441 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.222Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.938µs 2024-03-07T03:07:18.441 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.222Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-07T03:07:18.441 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.224Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-03-07T03:07:18.442 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.224Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-03-07T03:07:18.442 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.224Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=25.599µs wal_replay_duration=2.01763ms wbl_replay_duration=139ns total_replay_duration=2.064383ms 2024-03-07T03:07:18.442 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.225Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-07T03:07:18.442 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.226Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-07T03:07:18.442 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.226Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-07T03:07:18.442 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.246Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.760535ms db_storage=925ns remote_storage=1.434µs web_handler=442ns query_engine=643ns scrape=394.962µs scrape_sd=159.78µs notify=11.587µs notify_sd=10.037µs rules=19.854959ms tracing=6.401µs 2024-03-07T03:07:18.442 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.246Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-07T03:07:18.442 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[50256]: ts=2024-03-07T03:07:18.246Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-07T03:07:18.836 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=sqlstore t=2024-03-07T03:07:18.446743831Z level=info msg="Created default organization" 2024-03-07T03:07:18.836 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=plugin.loader t=2024-03-07T03:07:18.639123737Z level=info msg="Plugin registered" pluginID=input 2024-03-07T03:07:18.836 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=plugin.loader t=2024-03-07T03:07:18.678845587Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-03-07T03:07:18.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=plugin.loader t=2024-03-07T03:07:18.678875796Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-03-07T03:07:18.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=secrets t=2024-03-07T03:07:18.678919628Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-03-07T03:07:18.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=query_data t=2024-03-07T03:07:18.681231308Z level=info msg="Query Service initialization" 2024-03-07T03:07:18.837 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:18 smithi189 bash[49584]: logger=live.push_http t=2024-03-07T03:07:18.684356948Z level=info msg="Live Push Gateway initialization" 2024-03-07T03:07:20.142 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:19 smithi189 bash[49584]: logger=infra.usagestats.collector t=2024-03-07T03:07:19.872477339Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-03-07T03:07:20.142 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:19 smithi189 bash[49584]: logger=provisioning.datasources t=2024-03-07T03:07:19.929435939Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-03-07T03:07:20.143 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:20 smithi189 bash[49584]: logger=provisioning.datasources t=2024-03-07T03:07:20.024087366Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-03-07T03:07:20.351 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:07:20.351 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":64,"fsid":"8947a0f8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:55:42.039360+0000","modified":"2024-03-07T03:06:51.319603+0000","last_up_change":"2024-03-07T03:05:04.610834+0000","last_in_change":"2024-03-07T03:04:39.932769+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:00:59.846304+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-07T03:06:41.584515+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"242c63ee-0417-4ebf-bd71-6239f5d3634f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6803","nonce":1367316946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6805","nonce":1367316946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6809","nonce":1367316946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6807","nonce":1367316946}]},"public_addr":"172.21.15.145:6803/1367316946","cluster_addr":"172.21.15.145:6805/1367316946","heartbeat_back_addr":"172.21.15.145:6809/1367316946","heartbeat_front_addr":"172.21.15.145:6807/1367316946","state":["exists","up"]},{"osd":1,"uuid":"3c2818da-5d32-4f7a-8a63-8d7d679961e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6811","nonce":1873038359}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6813","nonce":1873038359}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6817","nonce":1873038359}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6815","nonce":1873038359}]},"public_addr":"172.21.15.145:6811/1873038359","cluster_addr":"172.21.15.145:6813/1873038359","heartbeat_back_addr":"172.21.15.145:6817/1873038359","heartbeat_front_addr":"172.21.15.145:6815/1873038359","state":["exists","up"]},{"osd":2,"uuid":"905ec219-dd9c-4f30-bdab-ed78d1405c11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6819","nonce":462216015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6821","nonce":462216015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6825","nonce":462216015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6823","nonce":462216015}]},"public_addr":"172.21.15.145:6819/462216015","cluster_addr":"172.21.15.145:6821/462216015","heartbeat_back_addr":"172.21.15.145:6825/462216015","heartbeat_front_addr":"172.21.15.145:6823/462216015","state":["exists","up"]},{"osd":3,"uuid":"5f466751-f5a7-4d18-87ab-96251b77b663","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6827","nonce":2306759084}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6829","nonce":2306759084}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6833","nonce":2306759084}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6831","nonce":2306759084}]},"public_addr":"172.21.15.145:6827/2306759084","cluster_addr":"172.21.15.145:6829/2306759084","heartbeat_back_addr":"172.21.15.145:6833/2306759084","heartbeat_front_addr":"172.21.15.145:6831/2306759084","state":["exists","up"]},{"osd":4,"uuid":"633f56af-3ff8-4954-97b4-bc8122bb10ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6801","nonce":3152992669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6803","nonce":3152992669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6807","nonce":3152992669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6805","nonce":3152992669}]},"public_addr":"172.21.15.189:6801/3152992669","cluster_addr":"172.21.15.189:6803/3152992669","heartbeat_back_addr":"172.21.15.189:6807/3152992669","heartbeat_front_addr":"172.21.15.189:6805/3152992669","state":["exists","up"]},{"osd":5,"uuid":"531d5127-60fe-4669-ab7d-fe4f5e5753cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6809","nonce":2096444201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6811","nonce":2096444201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6815","nonce":2096444201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6813","nonce":2096444201}]},"public_addr":"172.21.15.189:6809/2096444201","cluster_addr":"172.21.15.189:6811/2096444201","heartbeat_back_addr":"172.21.15.189:6815/2096444201","heartbeat_front_addr":"172.21.15.189:6813/2096444201","state":["exists","up"]},{"osd":6,"uuid":"8f2b456e-712e-45f1-8d84-0fdc4a6d860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6817","nonce":4219543665}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6819","nonce":4219543665}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6823","nonce":4219543665}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6821","nonce":4219543665}]},"public_addr":"172.21.15.189:6817/4219543665","cluster_addr":"172.21.15.189:6819/4219543665","heartbeat_back_addr":"172.21.15.189:6823/4219543665","heartbeat_front_addr":"172.21.15.189:6821/4219543665","state":["exists","up"]},{"osd":7,"uuid":"24dc7eba-4cd4-414d-93d2-598f5f2e655c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6825","nonce":452348965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6827","nonce":452348965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6831","nonce":452348965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6829","nonce":452348965}]},"public_addr":"172.21.15.189:6825/452348965","cluster_addr":"172.21.15.189:6827/452348965","heartbeat_back_addr":"172.21.15.189:6831/452348965","heartbeat_front_addr":"172.21.15.189:6829/452348965","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T02:59:11.112897+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:03.501748+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:56.635004+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:01:50.372735+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:38.150721+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:25.965239+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:14.345171+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:03.479901+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.145:6801/2891889777":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3299433409":"2024-03-08T03:06:02.331013+0000","172.21.15.145:6800/2891889777":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/2939737297":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/3852118874":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3488231986":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/2236140381":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/1978862755":"2024-03-08T03:06:02.331013+0000","172.21.15.145:6800/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/2696605014":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/131277059":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/2414597158":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:6800/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/321164949":"2024-03-08T02:56:14.659813+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:07:20.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:20 smithi189 bash[23614]: cluster 2024-03-07T03:07:18.377686+0000 mgr.y (mgr.24302) 72 : cluster 0 pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:20.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:20 smithi189 bash[49584]: logger=provisioning.alerting t=2024-03-07T03:07:20.147985372Z level=info msg="starting to provision alerting" 2024-03-07T03:07:20.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:20 smithi189 bash[49584]: logger=provisioning.alerting t=2024-03-07T03:07:20.148048128Z level=info msg="finished to provision alerting" 2024-03-07T03:07:20.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:20 smithi189 bash[49584]: logger=grafanaStorageLogger t=2024-03-07T03:07:20.14853982Z level=info msg="storage starting" 2024-03-07T03:07:20.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:20 smithi189 bash[49584]: logger=ngalert.state.manager t=2024-03-07T03:07:20.148588073Z level=info msg="Warming state cache for startup" 2024-03-07T03:07:20.587 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:20 smithi189 bash[49584]: logger=http.server t=2024-03-07T03:07:20.154298673Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-03-07T03:07:20.587 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:20 smithi189 bash[49584]: logger=ngalert.state.manager t=2024-03-07T03:07:20.2821061Z level=info msg="State cache has been initialized" states=0 duration=133.51437ms 2024-03-07T03:07:20.587 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:20 smithi189 bash[49584]: logger=ngalert.multiorg.alertmanager t=2024-03-07T03:07:20.282250618Z level=info msg="starting MultiOrg Alertmanager" 2024-03-07T03:07:20.587 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:20 smithi189 bash[49584]: logger=ticker t=2024-03-07T03:07:20.282299991Z level=info msg=starting first_tick=2024-03-07T03:07:30Z 2024-03-07T03:07:20.587 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:07:20 smithi189 bash[49584]: logger=sqlstore.transactions t=2024-03-07T03:07:20.370861311Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-07T03:07:20.623 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:20 smithi145 bash[21339]: cluster 2024-03-07T03:07:18.377686+0000 mgr.y (mgr.24302) 72 : cluster 0 pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:20.623 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:20 smithi145 bash[30906]: cluster 2024-03-07T03:07:18.377686+0000 mgr.y (mgr.24302) 72 : cluster 0 pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:21.137 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-07T03:07:21.138 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-07T03:07:21.138 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-07T03:07:21.139 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-07T03:07:21.139 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-07T03:07:21.139 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-07T03:07:21.140 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-07T03:07:21.140 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-07T03:07:21.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:21 smithi145 bash[30906]: audit 2024-03-07T03:07:20.355722+0000 mon.a (mon.0) 746 : audit 0 from='client.? 172.21.15.145:0/3236233796' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:07:21.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:21 smithi145 bash[21339]: audit 2024-03-07T03:07:20.355722+0000 mon.a (mon.0) 746 : audit 0 from='client.? 172.21.15.145:0/3236233796' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:07:21.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:21 smithi189 bash[23614]: audit 2024-03-07T03:07:20.355722+0000 mon.a (mon.0) 746 : audit 0 from='client.? 172.21.15.145:0/3236233796' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:07:22.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:22 smithi145 bash[21339]: cluster 2024-03-07T03:07:20.378115+0000 mgr.y (mgr.24302) 73 : cluster 0 pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:22.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:22 smithi145 bash[30906]: cluster 2024-03-07T03:07:20.378115+0000 mgr.y (mgr.24302) 73 : cluster 0 pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:22.566 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:07:22 smithi145 bash[57902]: ts=2024-03-07T03:07:22.154Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001922579s 2024-03-07T03:07:22.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:22 smithi189 bash[23614]: cluster 2024-03-07T03:07:20.378115+0000 mgr.y (mgr.24302) 73 : cluster 0 pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:24.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[21339]: cluster 2024-03-07T03:07:22.378600+0000 mgr.y (mgr.24302) 74 : cluster 0 pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:24.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[21339]: audit 2024-03-07T03:07:23.181329+0000 mon.a (mon.0) 747 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[21339]: audit 2024-03-07T03:07:23.191703+0000 mon.a (mon.0) 748 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[21339]: audit 2024-03-07T03:07:23.550221+0000 mon.a (mon.0) 749 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[21339]: audit 2024-03-07T03:07:23.567413+0000 mon.a (mon.0) 750 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[21339]: audit 2024-03-07T03:07:24.132640+0000 mon.c (mon.2) 61 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:24.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[21339]: audit 2024-03-07T03:07:24.134699+0000 mon.c (mon.2) 62 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:24.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[21339]: audit 2024-03-07T03:07:24.146245+0000 mon.a (mon.0) 751 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[30906]: cluster 2024-03-07T03:07:22.378600+0000 mgr.y (mgr.24302) 74 : cluster 0 pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:24.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[30906]: audit 2024-03-07T03:07:23.181329+0000 mon.a (mon.0) 747 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[30906]: audit 2024-03-07T03:07:23.191703+0000 mon.a (mon.0) 748 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[30906]: audit 2024-03-07T03:07:23.550221+0000 mon.a (mon.0) 749 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[30906]: audit 2024-03-07T03:07:23.567413+0000 mon.a (mon.0) 750 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[30906]: audit 2024-03-07T03:07:24.132640+0000 mon.c (mon.2) 61 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:24.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[30906]: audit 2024-03-07T03:07:24.134699+0000 mon.c (mon.2) 62 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:24.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:24 smithi145 bash[30906]: audit 2024-03-07T03:07:24.146245+0000 mon.a (mon.0) 751 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:24 smithi189 bash[23614]: cluster 2024-03-07T03:07:22.378600+0000 mgr.y (mgr.24302) 74 : cluster 0 pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:24.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:24 smithi189 bash[23614]: audit 2024-03-07T03:07:23.181329+0000 mon.a (mon.0) 747 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:24 smithi189 bash[23614]: audit 2024-03-07T03:07:23.191703+0000 mon.a (mon.0) 748 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:24 smithi189 bash[23614]: audit 2024-03-07T03:07:23.550221+0000 mon.a (mon.0) 749 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:24 smithi189 bash[23614]: audit 2024-03-07T03:07:23.567413+0000 mon.a (mon.0) 750 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:24.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:24 smithi189 bash[23614]: audit 2024-03-07T03:07:24.132640+0000 mon.c (mon.2) 61 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:24.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:24 smithi189 bash[23614]: audit 2024-03-07T03:07:24.134699+0000 mon.c (mon.2) 62 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:24.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:24 smithi189 bash[23614]: audit 2024-03-07T03:07:24.146245+0000 mon.a (mon.0) 751 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:25.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:25 smithi145 bash[21339]: audit 2024-03-07T03:07:24.234470+0000 mon.c (mon.2) 63 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:25.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:25 smithi145 bash[21339]: audit 2024-03-07T03:07:24.237435+0000 mon.c (mon.2) 64 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:25.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:25 smithi145 bash[21339]: audit 2024-03-07T03:07:24.238854+0000 mon.c (mon.2) 65 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:25.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:25 smithi145 bash[21339]: audit 2024-03-07T03:07:24.247251+0000 mon.a (mon.0) 752 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:25.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:25 smithi145 bash[30906]: audit 2024-03-07T03:07:24.234470+0000 mon.c (mon.2) 63 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:25.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:25 smithi145 bash[30906]: audit 2024-03-07T03:07:24.237435+0000 mon.c (mon.2) 64 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:25.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:25 smithi145 bash[30906]: audit 2024-03-07T03:07:24.238854+0000 mon.c (mon.2) 65 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:25.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:25 smithi145 bash[30906]: audit 2024-03-07T03:07:24.247251+0000 mon.a (mon.0) 752 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:25.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:25 smithi189 bash[23614]: audit 2024-03-07T03:07:24.234470+0000 mon.c (mon.2) 63 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:25.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:25 smithi189 bash[23614]: audit 2024-03-07T03:07:24.237435+0000 mon.c (mon.2) 64 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:25.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:25 smithi189 bash[23614]: audit 2024-03-07T03:07:24.238854+0000 mon.c (mon.2) 65 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:25.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:25 smithi189 bash[23614]: audit 2024-03-07T03:07:24.247251+0000 mon.a (mon.0) 752 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:07:25.689 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:25.690 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:25.690 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:25.694 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:25.694 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:25.694 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:25.695 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:25.695 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:26.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:26 smithi189 bash[23614]: cluster 2024-03-07T03:07:24.379306+0000 mgr.y (mgr.24302) 75 : cluster 0 pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:26.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:26 smithi145 bash[30906]: cluster 2024-03-07T03:07:24.379306+0000 mgr.y (mgr.24302) 75 : cluster 0 pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:26.634 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:26 smithi145 bash[21339]: cluster 2024-03-07T03:07:24.379306+0000 mgr.y (mgr.24302) 75 : cluster 0 pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:28.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:28 smithi145 bash[21339]: cluster 2024-03-07T03:07:26.379877+0000 mgr.y (mgr.24302) 76 : cluster 0 pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:28.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:28 smithi145 bash[30906]: cluster 2024-03-07T03:07:26.379877+0000 mgr.y (mgr.24302) 76 : cluster 0 pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:28.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:28 smithi189 bash[23614]: cluster 2024-03-07T03:07:26.379877+0000 mgr.y (mgr.24302) 76 : cluster 0 pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:30.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:30 smithi189 bash[23614]: cluster 2024-03-07T03:07:28.380470+0000 mgr.y (mgr.24302) 77 : cluster 0 pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:30.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:30 smithi145 bash[21339]: cluster 2024-03-07T03:07:28.380470+0000 mgr.y (mgr.24302) 77 : cluster 0 pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:30.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:30 smithi145 bash[30906]: cluster 2024-03-07T03:07:28.380470+0000 mgr.y (mgr.24302) 77 : cluster 0 pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:32.494 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:32 smithi145 bash[21339]: cluster 2024-03-07T03:07:30.381156+0000 mgr.y (mgr.24302) 78 : cluster 0 pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:32.494 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:32 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:07:32] "GET /metrics HTTP/1.1" 200 28557 "" "Prometheus/2.43.0" 2024-03-07T03:07:32.494 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:32 smithi145 bash[30906]: cluster 2024-03-07T03:07:30.381156+0000 mgr.y (mgr.24302) 78 : cluster 0 pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:32.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:32 smithi189 bash[23614]: cluster 2024-03-07T03:07:30.381156+0000 mgr.y (mgr.24302) 78 : cluster 0 pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:33.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:33 smithi145 bash[21339]: audit 2024-03-07T03:07:32.441115+0000 mon.c (mon.2) 66 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:07:33.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:33 smithi145 bash[30906]: audit 2024-03-07T03:07:32.441115+0000 mon.c (mon.2) 66 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:07:33.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:33 smithi189 bash[23614]: audit 2024-03-07T03:07:32.441115+0000 mon.c (mon.2) 66 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:07:34.702 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:34 smithi145 bash[21339]: cluster 2024-03-07T03:07:32.381642+0000 mgr.y (mgr.24302) 79 : cluster 0 pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:34.702 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:34 smithi145 bash[30906]: cluster 2024-03-07T03:07:32.381642+0000 mgr.y (mgr.24302) 79 : cluster 0 pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:34.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:34 smithi189 bash[23614]: cluster 2024-03-07T03:07:32.381642+0000 mgr.y (mgr.24302) 79 : cluster 0 pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:36.711 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:36 smithi145 bash[21339]: cluster 2024-03-07T03:07:34.382317+0000 mgr.y (mgr.24302) 80 : cluster 0 pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:36.712 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:36 smithi145 bash[30906]: cluster 2024-03-07T03:07:34.382317+0000 mgr.y (mgr.24302) 80 : cluster 0 pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:36.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:36 smithi189 bash[23614]: cluster 2024-03-07T03:07:34.382317+0000 mgr.y (mgr.24302) 80 : cluster 0 pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:38.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:38 smithi145 bash[21339]: cluster 2024-03-07T03:07:36.382822+0000 mgr.y (mgr.24302) 81 : cluster 0 pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:38.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:38 smithi145 bash[30906]: cluster 2024-03-07T03:07:36.382822+0000 mgr.y (mgr.24302) 81 : cluster 0 pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:38.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:38 smithi189 bash[23614]: cluster 2024-03-07T03:07:36.382822+0000 mgr.y (mgr.24302) 81 : cluster 0 pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:40.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:40 smithi145 bash[21339]: cluster 2024-03-07T03:07:38.383373+0000 mgr.y (mgr.24302) 82 : cluster 0 pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:40.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:40 smithi145 bash[30906]: cluster 2024-03-07T03:07:38.383373+0000 mgr.y (mgr.24302) 82 : cluster 0 pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:40.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:40 smithi189 bash[23614]: cluster 2024-03-07T03:07:38.383373+0000 mgr.y (mgr.24302) 82 : cluster 0 pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:42.065 INFO:teuthology.orchestra.run.smithi145.stdout:236223201312 2024-03-07T03:07:42.066 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-07T03:07:42.233 INFO:teuthology.orchestra.run.smithi145.stdout:201863462955 2024-03-07T03:07:42.233 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-07T03:07:42.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:42 smithi145 bash[21339]: cluster 2024-03-07T03:07:40.383854+0000 mgr.y (mgr.24302) 83 : cluster 0 pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:42.816 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:42 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:07:42] "GET /metrics HTTP/1.1" 200 28552 "" "Prometheus/2.43.0" 2024-03-07T03:07:42.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:42 smithi145 bash[30906]: cluster 2024-03-07T03:07:40.383854+0000 mgr.y (mgr.24302) 83 : cluster 0 pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:42.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:42 smithi189 bash[23614]: cluster 2024-03-07T03:07:40.383854+0000 mgr.y (mgr.24302) 83 : cluster 0 pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:44.149 INFO:teuthology.orchestra.run.smithi145.stdout:34359738471 2024-03-07T03:07:44.149 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-07T03:07:44.439 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:44 smithi145 bash[21339]: cluster 2024-03-07T03:07:42.384166+0000 mgr.y (mgr.24302) 84 : cluster 0 pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:44.440 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:44 smithi145 bash[30906]: cluster 2024-03-07T03:07:42.384166+0000 mgr.y (mgr.24302) 84 : cluster 0 pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:44.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:44 smithi189 bash[23614]: cluster 2024-03-07T03:07:42.384166+0000 mgr.y (mgr.24302) 84 : cluster 0 pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:46.353 INFO:teuthology.orchestra.run.smithi145.stdout:55834574942 2024-03-07T03:07:46.354 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-07T03:07:46.425 INFO:teuthology.orchestra.run.smithi145.stdout:77309411412 2024-03-07T03:07:46.425 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-07T03:07:46.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:46 smithi145 bash[21339]: cluster 2024-03-07T03:07:44.384514+0000 mgr.y (mgr.24302) 85 : cluster 0 pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:46.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:46 smithi145 bash[30906]: cluster 2024-03-07T03:07:44.384514+0000 mgr.y (mgr.24302) 85 : cluster 0 pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:46.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:46 smithi189 bash[23614]: cluster 2024-03-07T03:07:44.384514+0000 mgr.y (mgr.24302) 85 : cluster 0 pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:47.698 INFO:teuthology.orchestra.run.smithi145.stdout:133143986239 2024-03-07T03:07:47.699 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-07T03:07:47.731 INFO:teuthology.orchestra.run.smithi145.stdout:111669149768 2024-03-07T03:07:47.732 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-07T03:07:47.743 INFO:teuthology.orchestra.run.smithi145.stdout:167503724597 2024-03-07T03:07:47.743 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-07T03:07:48.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:48 smithi145 bash[30906]: cluster 2024-03-07T03:07:46.384969+0000 mgr.y (mgr.24302) 86 : cluster 0 pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:48.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:48 smithi145 bash[30906]: audit 2024-03-07T03:07:47.442188+0000 mon.c (mon.2) 67 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:07:48.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:48 smithi145 bash[21339]: cluster 2024-03-07T03:07:46.384969+0000 mgr.y (mgr.24302) 86 : cluster 0 pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:48.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:48 smithi145 bash[21339]: audit 2024-03-07T03:07:47.442188+0000 mon.c (mon.2) 67 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:07:48.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:48 smithi189 bash[23614]: cluster 2024-03-07T03:07:46.384969+0000 mgr.y (mgr.24302) 86 : cluster 0 pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:48.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:48 smithi189 bash[23614]: audit 2024-03-07T03:07:47.442188+0000 mon.c (mon.2) 67 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:07:49.829 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:49.832 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:49.832 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:50.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:50 smithi145 bash[21339]: cluster 2024-03-07T03:07:48.385479+0000 mgr.y (mgr.24302) 87 : cluster 0 pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:50.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:50 smithi145 bash[30906]: cluster 2024-03-07T03:07:48.385479+0000 mgr.y (mgr.24302) 87 : cluster 0 pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:50 smithi189 bash[23614]: cluster 2024-03-07T03:07:48.385479+0000 mgr.y (mgr.24302) 87 : cluster 0 pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:50.846 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:50.846 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:51.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:51 smithi145 bash[21339]: cluster 2024-03-07T03:07:50.386038+0000 mgr.y (mgr.24302) 88 : cluster 0 pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:51.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:51 smithi145 bash[30906]: cluster 2024-03-07T03:07:50.386038+0000 mgr.y (mgr.24302) 88 : cluster 0 pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:51.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:51 smithi189 bash[23614]: cluster 2024-03-07T03:07:50.386038+0000 mgr.y (mgr.24302) 88 : cluster 0 pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:51.861 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:51.862 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:51.862 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:52.815 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:07:52 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:07:52] "GET /metrics HTTP/1.1" 200 28553 "" "Prometheus/2.43.0" 2024-03-07T03:07:54.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:54 smithi145 bash[21339]: cluster 2024-03-07T03:07:52.386549+0000 mgr.y (mgr.24302) 89 : cluster 0 pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:54.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:54 smithi145 bash[30906]: cluster 2024-03-07T03:07:52.386549+0000 mgr.y (mgr.24302) 89 : cluster 0 pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:54.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:54 smithi189 bash[23614]: cluster 2024-03-07T03:07:52.386549+0000 mgr.y (mgr.24302) 89 : cluster 0 pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:56.538 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:56 smithi145 bash[21339]: cluster 2024-03-07T03:07:54.387232+0000 mgr.y (mgr.24302) 90 : cluster 0 pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:56.539 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:56 smithi145 bash[30906]: cluster 2024-03-07T03:07:54.387232+0000 mgr.y (mgr.24302) 90 : cluster 0 pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:56.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:56 smithi189 bash[23614]: cluster 2024-03-07T03:07:54.387232+0000 mgr.y (mgr.24302) 90 : cluster 0 pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:58.592 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:07:58 smithi145 bash[21339]: cluster 2024-03-07T03:07:56.387859+0000 mgr.y (mgr.24302) 91 : cluster 0 pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:58.593 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:07:58 smithi145 bash[30906]: cluster 2024-03-07T03:07:56.387859+0000 mgr.y (mgr.24302) 91 : cluster 0 pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:58.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:07:58 smithi189 bash[23614]: cluster 2024-03-07T03:07:56.387859+0000 mgr.y (mgr.24302) 91 : cluster 0 pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:00.728 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:00 smithi145 bash[21339]: cluster 2024-03-07T03:07:58.388348+0000 mgr.y (mgr.24302) 92 : cluster 0 pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:00.729 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:00 smithi145 bash[30906]: cluster 2024-03-07T03:07:58.388348+0000 mgr.y (mgr.24302) 92 : cluster 0 pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:00.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:00 smithi189 bash[23614]: cluster 2024-03-07T03:07:58.388348+0000 mgr.y (mgr.24302) 92 : cluster 0 pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:02.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:02 smithi145 bash[21339]: cluster 2024-03-07T03:08:00.388986+0000 mgr.y (mgr.24302) 93 : cluster 0 pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:02.816 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:08:02 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:08:02] "GET /metrics HTTP/1.1" 200 28553 "" "Prometheus/2.43.0" 2024-03-07T03:08:02.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:02 smithi145 bash[30906]: cluster 2024-03-07T03:08:00.388986+0000 mgr.y (mgr.24302) 93 : cluster 0 pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:02.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:02 smithi189 bash[23614]: cluster 2024-03-07T03:08:00.388986+0000 mgr.y (mgr.24302) 93 : cluster 0 pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:03.423 INFO:teuthology.orchestra.run.smithi145.stdout:201863462959 2024-03-07T03:08:03.660 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:03 smithi145 bash[21339]: audit 2024-03-07T03:08:02.447952+0000 mon.c (mon.2) 68 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:03.660 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:03 smithi145 bash[30906]: audit 2024-03-07T03:08:02.447952+0000 mon.c (mon.2) 68 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:03.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:03 smithi189 bash[23614]: audit 2024-03-07T03:08:02.447952+0000 mon.c (mon.2) 68 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:04.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:04 smithi145 bash[21339]: cluster 2024-03-07T03:08:02.389544+0000 mgr.y (mgr.24302) 94 : cluster 0 pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:04.822 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:04 smithi145 bash[21339]: audit 2024-03-07T03:08:03.429446+0000 mon.a (mon.0) 753 : audit 0 from='client.? 172.21.15.145:0/1206786820' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:08:04.823 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:04 smithi145 bash[30906]: cluster 2024-03-07T03:08:02.389544+0000 mgr.y (mgr.24302) 94 : cluster 0 pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:04.823 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:04 smithi145 bash[30906]: audit 2024-03-07T03:08:03.429446+0000 mon.a (mon.0) 753 : audit 0 from='client.? 172.21.15.145:0/1206786820' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:08:04.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:04 smithi189 bash[23614]: cluster 2024-03-07T03:08:02.389544+0000 mgr.y (mgr.24302) 94 : cluster 0 pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:04.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:04 smithi189 bash[23614]: audit 2024-03-07T03:08:03.429446+0000 mon.a (mon.0) 753 : audit 0 from='client.? 172.21.15.145:0/1206786820' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:08:05.478 INFO:teuthology.orchestra.run.smithi145.stdout:34359738476 2024-03-07T03:08:05.901 INFO:teuthology.orchestra.run.smithi145.stdout:236223201318 2024-03-07T03:08:06.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:06 smithi145 bash[21339]: cluster 2024-03-07T03:08:04.390127+0000 mgr.y (mgr.24302) 95 : cluster 0 pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:06.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:06 smithi145 bash[21339]: audit 2024-03-07T03:08:05.483712+0000 mon.a (mon.0) 754 : audit 0 from='client.? 172.21.15.145:0/4232308530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:08:06.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:06 smithi145 bash[21339]: audit 2024-03-07T03:08:05.907333+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.145:0/4276664560' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:08:06.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:06 smithi145 bash[30906]: cluster 2024-03-07T03:08:04.390127+0000 mgr.y (mgr.24302) 95 : cluster 0 pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:06.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:06 smithi145 bash[30906]: audit 2024-03-07T03:08:05.483712+0000 mon.a (mon.0) 754 : audit 0 from='client.? 172.21.15.145:0/4232308530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:08:06.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:06 smithi145 bash[30906]: audit 2024-03-07T03:08:05.907333+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.145:0/4276664560' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:08:06.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:06 smithi189 bash[23614]: cluster 2024-03-07T03:08:04.390127+0000 mgr.y (mgr.24302) 95 : cluster 0 pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:06.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:06 smithi189 bash[23614]: audit 2024-03-07T03:08:05.483712+0000 mon.a (mon.0) 754 : audit 0 from='client.? 172.21.15.145:0/4232308530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:08:06.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:06 smithi189 bash[23614]: audit 2024-03-07T03:08:05.907333+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.145:0/4276664560' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:08:07.142 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462955 got 201863462959 for osd.6 2024-03-07T03:08:07.142 DEBUG:teuthology.parallel:result is None 2024-03-07T03:08:07.385 INFO:teuthology.orchestra.run.smithi145.stdout:111669149772 2024-03-07T03:08:07.494 INFO:teuthology.orchestra.run.smithi145.stdout:77309411416 2024-03-07T03:08:08.656 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:08 smithi145 bash[21339]: cluster 2024-03-07T03:08:06.390634+0000 mgr.y (mgr.24302) 96 : cluster 0 pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:08.656 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:08 smithi145 bash[21339]: audit 2024-03-07T03:08:07.391702+0000 mon.a (mon.0) 755 : audit 0 from='client.? 172.21.15.145:0/4135448775' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:08:08.656 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:08 smithi145 bash[21339]: audit 2024-03-07T03:08:07.500890+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.145:0/2476388005' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:08:08.657 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:08 smithi145 bash[30906]: cluster 2024-03-07T03:08:06.390634+0000 mgr.y (mgr.24302) 96 : cluster 0 pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:08.657 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:08 smithi145 bash[30906]: audit 2024-03-07T03:08:07.391702+0000 mon.a (mon.0) 755 : audit 0 from='client.? 172.21.15.145:0/4135448775' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:08:08.657 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:08 smithi145 bash[30906]: audit 2024-03-07T03:08:07.500890+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.145:0/2476388005' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:08:08.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:08 smithi189 bash[23614]: cluster 2024-03-07T03:08:06.390634+0000 mgr.y (mgr.24302) 96 : cluster 0 pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:08.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:08 smithi189 bash[23614]: audit 2024-03-07T03:08:07.391702+0000 mon.a (mon.0) 755 : audit 0 from='client.? 172.21.15.145:0/4135448775' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:08:08.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:08 smithi189 bash[23614]: audit 2024-03-07T03:08:07.500890+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.145:0/2476388005' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:08:08.972 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738471 got 34359738476 for osd.0 2024-03-07T03:08:08.972 DEBUG:teuthology.parallel:result is None 2024-03-07T03:08:09.050 INFO:teuthology.orchestra.run.smithi145.stdout:55834574946 2024-03-07T03:08:09.275 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201312 got 236223201318 for osd.7 2024-03-07T03:08:09.275 DEBUG:teuthology.parallel:result is None 2024-03-07T03:08:09.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:09 smithi145 bash[21339]: audit 2024-03-07T03:08:09.056601+0000 mon.a (mon.0) 756 : audit 0 from='client.? 172.21.15.145:0/2806705195' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:08:09.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:09 smithi145 bash[30906]: audit 2024-03-07T03:08:09.056601+0000 mon.a (mon.0) 756 : audit 0 from='client.? 172.21.15.145:0/2806705195' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:08:09.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:09 smithi189 bash[23614]: audit 2024-03-07T03:08:09.056601+0000 mon.a (mon.0) 756 : audit 0 from='client.? 172.21.15.145:0/2806705195' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:08:09.953 INFO:teuthology.orchestra.run.smithi145.stdout:167503724601 2024-03-07T03:08:10.688 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:10 smithi145 bash[21339]: cluster 2024-03-07T03:08:08.391115+0000 mgr.y (mgr.24302) 97 : cluster 0 pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:10.688 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:10 smithi145 bash[21339]: audit 2024-03-07T03:08:09.959061+0000 mon.a (mon.0) 757 : audit 0 from='client.? 172.21.15.145:0/3981536590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:08:10.688 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:10 smithi145 bash[30906]: cluster 2024-03-07T03:08:08.391115+0000 mgr.y (mgr.24302) 97 : cluster 0 pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:10.688 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:10 smithi145 bash[30906]: audit 2024-03-07T03:08:09.959061+0000 mon.a (mon.0) 757 : audit 0 from='client.? 172.21.15.145:0/3981536590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:08:10.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:10 smithi189 bash[23614]: cluster 2024-03-07T03:08:08.391115+0000 mgr.y (mgr.24302) 97 : cluster 0 pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:10.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:10 smithi189 bash[23614]: audit 2024-03-07T03:08:09.959061+0000 mon.a (mon.0) 757 : audit 0 from='client.? 172.21.15.145:0/3981536590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:08:11.058 INFO:teuthology.orchestra.run.smithi145.stdout:133143986243 2024-03-07T03:08:11.412 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149768 got 111669149772 for osd.3 2024-03-07T03:08:11.412 DEBUG:teuthology.parallel:result is None 2024-03-07T03:08:11.742 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:11 smithi145 bash[21339]: cluster 2024-03-07T03:08:10.391924+0000 mgr.y (mgr.24302) 98 : cluster 0 pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:11.742 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:11 smithi145 bash[21339]: audit 2024-03-07T03:08:11.064942+0000 mon.a (mon.0) 758 : audit 0 from='client.? 172.21.15.145:0/1196717457' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:08:11.742 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:11 smithi145 bash[30906]: cluster 2024-03-07T03:08:10.391924+0000 mgr.y (mgr.24302) 98 : cluster 0 pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:11.742 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:11 smithi145 bash[30906]: audit 2024-03-07T03:08:11.064942+0000 mon.a (mon.0) 758 : audit 0 from='client.? 172.21.15.145:0/1196717457' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:08:11.743 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411412 got 77309411416 for osd.2 2024-03-07T03:08:11.743 DEBUG:teuthology.parallel:result is None 2024-03-07T03:08:11.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:11 smithi189 bash[23614]: cluster 2024-03-07T03:08:10.391924+0000 mgr.y (mgr.24302) 98 : cluster 0 pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:11.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:11 smithi189 bash[23614]: audit 2024-03-07T03:08:11.064942+0000 mon.a (mon.0) 758 : audit 0 from='client.? 172.21.15.145:0/1196717457' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:08:12.402 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574942 got 55834574946 for osd.1 2024-03-07T03:08:12.403 DEBUG:teuthology.parallel:result is None 2024-03-07T03:08:12.815 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:08:12 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:08:12] "GET /metrics HTTP/1.1" 200 28554 "" "Prometheus/2.43.0" 2024-03-07T03:08:13.062 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724597 got 167503724601 for osd.5 2024-03-07T03:08:13.062 DEBUG:teuthology.parallel:result is None 2024-03-07T03:08:13.276 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986239 got 133143986243 for osd.4 2024-03-07T03:08:13.277 DEBUG:teuthology.parallel:result is None 2024-03-07T03:08:13.277 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-07T03:08:13.277 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph pg dump --format=json 2024-03-07T03:08:14.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:14 smithi145 bash[21339]: cluster 2024-03-07T03:08:12.392556+0000 mgr.y (mgr.24302) 99 : cluster 0 pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:14.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:14 smithi145 bash[30906]: cluster 2024-03-07T03:08:12.392556+0000 mgr.y (mgr.24302) 99 : cluster 0 pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:14.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:14 smithi189 bash[23614]: cluster 2024-03-07T03:08:12.392556+0000 mgr.y (mgr.24302) 99 : cluster 0 pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:16.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:16 smithi145 bash[21339]: cluster 2024-03-07T03:08:14.393195+0000 mgr.y (mgr.24302) 100 : cluster 0 pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:16.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:16 smithi145 bash[30906]: cluster 2024-03-07T03:08:14.393195+0000 mgr.y (mgr.24302) 100 : cluster 0 pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:16.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:16 smithi189 bash[23614]: cluster 2024-03-07T03:08:14.393195+0000 mgr.y (mgr.24302) 100 : cluster 0 pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:18.002 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:08:18.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:18 smithi145 bash[21339]: cluster 2024-03-07T03:08:16.393767+0000 mgr.y (mgr.24302) 101 : cluster 0 pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:18.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:18 smithi145 bash[21339]: audit 2024-03-07T03:08:17.448428+0000 mon.c (mon.2) 71 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:18.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:18 smithi145 bash[30906]: cluster 2024-03-07T03:08:16.393767+0000 mgr.y (mgr.24302) 101 : cluster 0 pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:18.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:18 smithi145 bash[30906]: audit 2024-03-07T03:08:17.448428+0000 mon.c (mon.2) 71 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:18.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:18 smithi189 bash[23614]: cluster 2024-03-07T03:08:16.393767+0000 mgr.y (mgr.24302) 101 : cluster 0 pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:18.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:18 smithi189 bash[23614]: audit 2024-03-07T03:08:17.448428+0000 mon.c (mon.2) 71 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:20.786 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:20 smithi145 bash[21339]: cluster 2024-03-07T03:08:18.394336+0000 mgr.y (mgr.24302) 102 : cluster 0 pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:20.787 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:20 smithi145 bash[30906]: cluster 2024-03-07T03:08:18.394336+0000 mgr.y (mgr.24302) 102 : cluster 0 pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:20.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:20 smithi189 bash[23614]: cluster 2024-03-07T03:08:18.394336+0000 mgr.y (mgr.24302) 102 : cluster 0 pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:21.184 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:08:21.185 INFO:teuthology.orchestra.run.smithi145.stderr:dumped all 2024-03-07T03:08:21.964 INFO:teuthology.orchestra.run.smithi145.stdout:{"pg_ready":true,"pg_map":{"version":76,"stamp":"2024-03-07T03:08:20.394574+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":222896,"kb_used_data":6736,"kb_used_omap":13,"kb_used_meta":216050,"kb_avail":749738320,"statfs":{"total":767960285184,"available":767732039680,"internally_reserved":0,"allocated":6897664,"data_stored":3903531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14248,"internal_metadata":221235288},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003728"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:44.348056+0000","last_change":"2024-03-07T03:06:42.667758+0000","last_active":"2024-03-07T03:06:44.348056+0000","last_peered":"2024-03-07T03:06:44.348056+0000","last_clean":"2024-03-07T03:06:44.348056+0000","last_became_active":"2024-03-07T03:06:42.667171+0000","last_became_peered":"2024-03-07T03:06:42.667171+0000","last_unstale":"2024-03-07T03:06:44.348056+0000","last_undegraded":"2024-03-07T03:06:44.348056+0000","last_fullsized":"2024-03-07T03:06:44.348056+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:55:07.432703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:43.669050+0000","last_change":"2024-03-07T03:06:42.794071+0000","last_active":"2024-03-07T03:06:43.669050+0000","last_peered":"2024-03-07T03:06:43.669050+0000","last_clean":"2024-03-07T03:06:43.669050+0000","last_became_active":"2024-03-07T03:06:42.792899+0000","last_became_peered":"2024-03-07T03:06:42.792899+0000","last_unstale":"2024-03-07T03:06:43.669050+0000","last_undegraded":"2024-03-07T03:06:43.669050+0000","last_fullsized":"2024-03-07T03:06:43.669050+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:07:21.918130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:43.670025+0000","last_change":"2024-03-07T03:06:42.792519+0000","last_active":"2024-03-07T03:06:43.670025+0000","last_peered":"2024-03-07T03:06:43.670025+0000","last_clean":"2024-03-07T03:06:43.670025+0000","last_became_active":"2024-03-07T03:06:42.792136+0000","last_became_peered":"2024-03-07T03:06:42.792136+0000","last_unstale":"2024-03-07T03:06:43.670025+0000","last_undegraded":"2024-03-07T03:06:43.670025+0000","last_fullsized":"2024-03-07T03:06:43.670025+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:50:03.686748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:43.669592+0000","last_change":"2024-03-07T03:06:42.668416+0000","last_active":"2024-03-07T03:06:43.669592+0000","last_peered":"2024-03-07T03:06:43.669592+0000","last_clean":"2024-03-07T03:06:43.669592+0000","last_became_active":"2024-03-07T03:06:42.668043+0000","last_became_peered":"2024-03-07T03:06:42.668043+0000","last_unstale":"2024-03-07T03:06:43.669592+0000","last_undegraded":"2024-03-07T03:06:43.669592+0000","last_fullsized":"2024-03-07T03:06:43.669592+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:07:59.168239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":260,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-03-07T03:06:52.849844+0000","last_change":"2024-03-07T03:05:08.667040+0000","last_active":"2024-03-07T03:06:52.849844+0000","last_peered":"2024-03-07T03:06:52.849844+0000","last_clean":"2024-03-07T03:06:52.849844+0000","last_became_active":"2024-03-07T03:05:08.665926+0000","last_became_peered":"2024-03-07T03:05:08.665926+0000","last_unstale":"2024-03-07T03:06:52.849844+0000","last_undegraded":"2024-03-07T03:06:52.849844+0000","last_fullsized":"2024-03-07T03:06:52.849844+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:01:00.212577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:01:00.212577+0000","last_clean_scrub_stamp":"2024-03-07T03:01:00.212577+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:11:34.307802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:44.849273+0000","last_change":"2024-03-07T03:06:42.670236+0000","last_active":"2024-03-07T03:06:44.849273+0000","last_peered":"2024-03-07T03:06:44.849273+0000","last_clean":"2024-03-07T03:06:44.849273+0000","last_became_active":"2024-03-07T03:06:42.669808+0000","last_became_peered":"2024-03-07T03:06:42.669808+0000","last_unstale":"2024-03-07T03:06:44.849273+0000","last_undegraded":"2024-03-07T03:06:44.849273+0000","last_fullsized":"2024-03-07T03:06:44.849273+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:42:36.928386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:43.899381+0000","last_change":"2024-03-07T03:06:42.795963+0000","last_active":"2024-03-07T03:06:43.899381+0000","last_peered":"2024-03-07T03:06:43.899381+0000","last_clean":"2024-03-07T03:06:43.899381+0000","last_became_active":"2024-03-07T03:06:42.795033+0000","last_became_peered":"2024-03-07T03:06:42.795033+0000","last_unstale":"2024-03-07T03:06:43.899381+0000","last_undegraded":"2024-03-07T03:06:43.899381+0000","last_fullsized":"2024-03-07T03:06:43.899381+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:12:44.845051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:44.849204+0000","last_change":"2024-03-07T03:06:42.667215+0000","last_active":"2024-03-07T03:06:44.849204+0000","last_peered":"2024-03-07T03:06:44.849204+0000","last_clean":"2024-03-07T03:06:44.849204+0000","last_became_active":"2024-03-07T03:06:42.666675+0000","last_became_peered":"2024-03-07T03:06:42.666675+0000","last_unstale":"2024-03-07T03:06:44.849204+0000","last_undegraded":"2024-03-07T03:06:44.849204+0000","last_fullsized":"2024-03-07T03:06:44.849204+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:35:31.916960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:43.669078+0000","last_change":"2024-03-07T03:06:42.666671+0000","last_active":"2024-03-07T03:06:43.669078+0000","last_peered":"2024-03-07T03:06:43.669078+0000","last_clean":"2024-03-07T03:06:43.669078+0000","last_became_active":"2024-03-07T03:06:42.666402+0000","last_became_peered":"2024-03-07T03:06:42.666402+0000","last_unstale":"2024-03-07T03:06:43.669078+0000","last_undegraded":"2024-03-07T03:06:43.669078+0000","last_fullsized":"2024-03-07T03:06:43.669078+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:28:33.638276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201321,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28224,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716928,"statfs":{"total":95995035648,"available":95966134272,"internally_reserved":0,"allocated":1232896,"data_stored":856851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.93600000000000005,"15min":0.92400000000000004},"min":{"1min":0.60199999999999998,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.1910000000000001,"5min":1.524,"15min":1.524},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1,"15min":1.03},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.056}]},{"osd":1,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.92000000000000004,"15min":0.93000000000000005},"min":{"1min":0.59499999999999997,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.2430000000000001,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.86299999999999999},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.0349999999999999,"15min":1.0580000000000001},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.3,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":1.282}]},{"osd":2,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.016,"15min":1.048},"min":{"1min":0.72899999999999998,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.1459999999999999,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.98799999999999999,"15min":0.98599999999999999},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.1990000000000001,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":0.97799999999999998}]},{"osd":3,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.92900000000000005,"15min":0.94399999999999995},"min":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.327,"5min":1.327,"15min":1.327},"last":1.018},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.0569999999999999,"15min":1.079},"min":{"1min":0.68700000000000006,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.2450000000000001,"5min":1.403,"15min":1.403},"last":1.181}]},{"osd":4,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.93000000000000005,"15min":0.92000000000000004},"min":{"1min":0.70499999999999996,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.2709999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.032,"5min":0.98499999999999999,"15min":0.98899999999999999},"min":{"1min":0.73299999999999998,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.73999999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.98899999999999999,"15min":0.98499999999999999},"min":{"1min":0.79000000000000004,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.2270000000000001,"5min":1.446,"15min":1.446},"last":1.0920000000000001},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.0920000000000001,"15min":1.1140000000000001},"min":{"1min":0.75800000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":1.1479999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.125,"15min":1.135},"min":{"1min":0.86499999999999999,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3600000000000001,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":1.224},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.1220000000000001,"15min":1.1200000000000001},"min":{"1min":0.94199999999999995,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.327,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.258}]}]},{"osd":6,"up_from":47,"seq":201863462963,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28220,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716932,"statfs":{"total":95995035648,"available":95966138368,"internally_reserved":0,"allocated":1228800,"data_stored":856578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.89300000000000002,"15min":0.92300000000000004},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.349,"5min":1.349,"15min":1.349},"last":0.98399999999999999},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.91100000000000003,"15min":0.91300000000000003},"min":{"1min":0.60899999999999999,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.496,"5min":1.496,"15min":1.496},"last":0.85199999999999998}]},{"osd":1,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.91500000000000004,"15min":0.90800000000000003},"min":{"1min":0.64900000000000002,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.4510000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.95899999999999996},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.91400000000000003,"15min":0.92600000000000005},"min":{"1min":0.47999999999999998,"5min":0.438,"15min":0.438},"max":{"1min":1.454,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":0.92300000000000004}]},{"osd":2,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.91500000000000004,"15min":0.90500000000000003},"min":{"1min":0.60799999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.3420000000000001,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":1.1619999999999999},{"interface":"front","average":{"1min":1.1870000000000001,"5min":0.98399999999999999,"15min":0.96699999999999997},"min":{"1min":0.63100000000000001,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":3.6360000000000001,"5min":3.6360000000000001,"15min":3.6360000000000001},"last":1.087}]},{"osd":3,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":0.96699999999999997,"15min":0.94899999999999995},"min":{"1min":0.70999999999999996,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":3.9169999999999998,"5min":3.9169999999999998,"15min":3.9169999999999998},"last":1.038},{"interface":"front","average":{"1min":1.21,"5min":1.02,"15min":1.004},"min":{"1min":0.64500000000000002,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":3.4790000000000001,"5min":3.4790000000000001,"15min":3.4790000000000001},"last":1.1200000000000001}]},{"osd":4,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.86199999999999999,"15min":0.81999999999999995},"min":{"1min":0.58399999999999996,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.0339999999999998,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.77400000000000002},{"interface":"front","average":{"1min":1.103,"5min":0.91300000000000003,"15min":0.89100000000000001},"min":{"1min":0.55900000000000005,"5min":0.307,"15min":0.307},"max":{"1min":1.7250000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":0.81299999999999994}]},{"osd":5,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":0.94699999999999995,"15min":0.90500000000000003},"min":{"1min":0.746,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.1720000000000002,"5min":2.1720000000000002,"15min":2.1720000000000002},"last":1.01},{"interface":"front","average":{"1min":1.121,"5min":0.98299999999999998,"15min":0.96499999999999997},"min":{"1min":0.51900000000000002,"5min":0.374,"15min":0.374},"max":{"1min":1.587,"5min":1.587,"15min":1.587},"last":0.72999999999999998}]},{"osd":7,"last update":"Thu Mar 7 03:08:14 2024","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.042,"15min":1.022},"min":{"1min":0.84299999999999997,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.3069999999999999,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.105,"5min":1.1180000000000001,"15min":1.0900000000000001},"min":{"1min":0.85599999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.3100000000000001,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":1.0649999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574949,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:07:20 2024","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.67100000000000004,"15min":0.58199999999999996},"min":{"1min":0.33200000000000002,"5min":0.246,"15min":0.246},"max":{"1min":1.0349999999999999,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.76800000000000002,"15min":0.60599999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.28899999999999998},"max":{"1min":1.2010000000000001,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":0.73099999999999998}]},{"osd":2,"last update":"Thu Mar 7 03:08:11 2024","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.76800000000000002,"15min":0.66400000000000003},"min":{"1min":0.36899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.01,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.76300000000000001,"15min":0.64800000000000002},"min":{"1min":0.40500000000000003,"5min":0.308,"15min":0.308},"max":{"1min":1.0469999999999999,"5min":1.327,"15min":1.327},"last":1.1850000000000001}]},{"osd":3,"last update":"Thu Mar 7 03:08:08 2024","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.83899999999999997,"15min":0.81100000000000005},"min":{"1min":0.40500000000000003,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":1.0629999999999999},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.78900000000000003,"15min":0.78100000000000003},"min":{"1min":0.36899999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.0640000000000001,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.84199999999999997}]},{"osd":4,"last update":"Thu Mar 7 03:07:49 2024","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.94099999999999995,"15min":0.93100000000000005},"min":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.3109999999999999,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.23},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.97999999999999998,"15min":0.92400000000000004},"min":{"1min":0.53800000000000003,"5min":0.503,"15min":0.503},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":1.27}]},{"osd":5,"last update":"Thu Mar 7 03:07:33 2024","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.97599999999999998,"15min":0.95799999999999996},"min":{"1min":0.65700000000000003,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.427,"5min":1.427,"15min":1.427},"last":1.1120000000000001},{"interface":"front","average":{"1min":1.036,"5min":1.0549999999999999,"15min":1.0429999999999999},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.4470000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.0169999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:07:22 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.089,"15min":1.089},"min":{"1min":0.61499999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.5109999999999999,"5min":1.8520000000000001,"15min":1.8520000000000001},"last":1.3660000000000001},{"interface":"front","average":{"1min":1.081,"5min":1.101,"15min":1.1120000000000001},"min":{"1min":0.67300000000000004,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.4990000000000001,"5min":1.512,"15min":1.512},"last":0.93200000000000005}]},{"osd":7,"last update":"Thu Mar 7 03:08:11 2024","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.145,"15min":1.177},"min":{"1min":0.67400000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.401,"5min":1.619,"15min":1.619},"last":1.151},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.133,"15min":1.135},"min":{"1min":0.71299999999999997,"5min":0.622,"15min":0.622},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":1.3180000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738479,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28228,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716924,"statfs":{"total":95995035648,"available":95966130176,"internally_reserved":0,"allocated":1236992,"data_stored":857182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Mar 7 03:07:24 2024","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.71299999999999997,"15min":0.58499999999999996},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.25},"max":{"1min":1.3340000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.77900000000000003,"15min":0.63600000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.25900000000000001},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.70899999999999996}]},{"osd":2,"last update":"Thu Mar 7 03:08:03 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.77900000000000003,"15min":0.67700000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.28499999999999998},"max":{"1min":0.94199999999999995,"5min":1.248,"15min":1.2869999999999999},"last":0.95699999999999996},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.81799999999999995,"15min":0.71699999999999997},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":1.177,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":1.0269999999999999}]},{"osd":3,"last update":"Thu Mar 7 03:08:03 2024","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.873,"15min":0.80900000000000005},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.175,"5min":1.6180000000000001,"15min":1.6180000000000001},"last":0.877},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.89300000000000002,"15min":0.82099999999999995},"min":{"1min":0.32200000000000001,"5min":0.316,"15min":0.316},"max":{"1min":1.1970000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.82499999999999996}]},{"osd":4,"last update":"Thu Mar 7 03:07:52 2024","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.97899999999999998,"15min":0.95599999999999996},"min":{"1min":0.64400000000000002,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.3680000000000001,"5min":1.375,"15min":1.375},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.91600000000000004,"15min":0.91000000000000003},"min":{"1min":0.59799999999999998,"5min":0.498,"15min":0.498},"max":{"1min":1.2649999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.77400000000000002}]},{"osd":5,"last update":"Thu Mar 7 03:07:33 2024","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.0349999999999999,"15min":0.98999999999999999},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.8109999999999999,"5min":1.8109999999999999,"15min":1.8109999999999999},"last":1.1559999999999999},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.0369999999999999,"15min":1.0009999999999999},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.6579999999999999,"5min":1.6579999999999999,"15min":1.6579999999999999},"last":0.98899999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:07:21 2024","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":1.0129999999999999,"15min":1.02},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.2589999999999999,"5min":1.2589999999999999,"15min":1.2589999999999999},"last":1.054},{"interface":"front","average":{"1min":1.1719999999999999,"5min":1.1819999999999999,"15min":1.196},"min":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.621,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.2210000000000001}]},{"osd":7,"last update":"Thu Mar 7 03:08:13 2024","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.0489999999999999,"15min":1.0609999999999999},"min":{"1min":0.80300000000000005,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":1.1279999999999999},{"interface":"front","average":{"1min":1.135,"5min":1.1759999999999999,"15min":1.1919999999999999},"min":{"1min":0.70299999999999996,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.724,"5min":1.724,"15min":1.724},"last":1.2729999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411419,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:08:12 2024","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.69999999999999996,"15min":0.61899999999999999},"min":{"1min":0.35099999999999998,"5min":0.318,"15min":0.307},"max":{"1min":1.153,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.96999999999999997},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.76100000000000001,"15min":0.66000000000000003},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.292,"5min":1.292,"15min":1.292},"last":0.751}]},{"osd":1,"last update":"Thu Mar 7 03:08:12 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.72899999999999998,"15min":0.67100000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":0.877,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.77800000000000002,"15min":0.70199999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.28999999999999998},"max":{"1min":1.0049999999999999,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.79600000000000004}]},{"osd":3,"last update":"Thu Mar 7 03:08:12 2024","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.83199999999999996,"15min":0.80700000000000005},"min":{"1min":0.38100000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.042,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.82299999999999995,"15min":0.80000000000000004},"min":{"1min":0.41499999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.1799999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.93500000000000005}]},{"osd":4,"last update":"Thu Mar 7 03:07:45 2024","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.97999999999999998,"15min":0.95399999999999996},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":1.026},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.95499999999999996,"15min":0.92400000000000004},"min":{"1min":0.57799999999999996,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.649,"5min":1.649,"15min":1.649},"last":1.232}]},{"osd":5,"last update":"Thu Mar 7 03:07:37 2024","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.98399999999999999,"15min":0.97099999999999997},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.1699999999999999,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.0800000000000001},{"interface":"front","average":{"1min":1.0249999999999999,"5min":0.98799999999999999,"15min":0.98499999999999999},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.53,"5min":1.53,"15min":1.53},"last":1.3009999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:07:22 2024","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.056,"15min":1.075},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.599,"5min":1.599,"15min":1.599},"last":1.4399999999999999},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.1120000000000001,"15min":1.1180000000000001},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.746,"5min":1.746,"15min":1.746},"last":1.395}]},{"osd":7,"last update":"Thu Mar 7 03:08:12 2024","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.0980000000000001,"15min":1.095},"min":{"1min":0.753,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.407,"5min":1.6759999999999999,"15min":1.6759999999999999},"last":1.208},{"interface":"front","average":{"1min":1.111,"5min":1.1559999999999999,"15min":1.155},"min":{"1min":0.56499999999999995,"5min":0.501,"15min":0.501},"max":{"1min":1.3899999999999999,"5min":1.696,"15min":1.696},"last":1.3520000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149775,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:08:06 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.66000000000000003,"15min":0.67900000000000005},"min":{"1min":0.29599999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.1759999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":1.133},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.753,"15min":0.73999999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.89700000000000002,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.70499999999999996}]},{"osd":1,"last update":"Thu Mar 7 03:08:06 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.77100000000000002,"15min":0.78500000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.165,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.879},{"interface":"front","average":{"1min":0.625,"5min":0.746,"15min":0.748},"min":{"1min":0.35299999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.97399999999999998,"5min":1.476,"15min":1.476},"last":0.73899999999999999}]},{"osd":2,"last update":"Thu Mar 7 03:08:06 2024","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.80100000000000005,"15min":0.81200000000000006},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.089,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.79500000000000004,"15min":0.76800000000000002},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.96699999999999997,"5min":1.407,"15min":1.407},"last":0.84299999999999997}]},{"osd":4,"last update":"Thu Mar 7 03:07:49 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.95199999999999996,"15min":0.89500000000000002},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.329,"5min":1.361,"15min":1.361},"last":0.97299999999999998},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.97199999999999998,"15min":0.94399999999999995},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.4330000000000001,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":1.103}]},{"osd":5,"last update":"Thu Mar 7 03:07:37 2024","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.96099999999999997,"15min":0.96499999999999997},"min":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.224,"5min":1.468,"15min":1.468},"last":1.1699999999999999},{"interface":"front","average":{"1min":0.97799999999999998,"5min":1.0529999999999999,"15min":1.081},"min":{"1min":0.61799999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.3360000000000001,"5min":1.492,"15min":1.492},"last":1.202}]},{"osd":6,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.0620000000000001,"15min":1.0720000000000001},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.431,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.0569999999999999,"15min":1.0620000000000001},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.5589999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":1.3360000000000001}]},{"osd":7,"last update":"Thu Mar 7 03:08:06 2024","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.0720000000000001,"15min":1.1000000000000001},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.3080000000000001,"5min":1.45,"15min":1.45},"last":1.0740000000000001},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.1040000000000001,"15min":1.1220000000000001},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.2929999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.2689999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986246,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:07:46 2024","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.84299999999999997,"15min":0.85899999999999999},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.198,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.90300000000000002},{"interface":"front","average":{"1min":0.871,"5min":0.871,"15min":0.85499999999999998},"min":{"1min":0.54000000000000004,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.226,"5min":1.323,"15min":1.323},"last":1.252}]},{"osd":1,"last update":"Thu Mar 7 03:07:46 2024","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.86599999999999999,"15min":0.86799999999999999},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.1919999999999999,"5min":1.194,"15min":1.194},"last":1.1020000000000001},{"interface":"front","average":{"1min":0.88600000000000001,"5min":0.89200000000000002,"15min":0.88900000000000001},"min":{"1min":0.52700000000000002,"5min":0.498,"15min":0.498},"max":{"1min":1.27,"5min":1.345,"15min":1.345},"last":1.2070000000000001}]},{"osd":2,"last update":"Thu Mar 7 03:07:46 2024","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.92300000000000004,"15min":0.90500000000000003},"min":{"1min":0.59099999999999997,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.264,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":1.1299999999999999},{"interface":"front","average":{"1min":0.871,"5min":0.88700000000000001,"15min":0.85499999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.1220000000000001,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.1519999999999999}]},{"osd":3,"last update":"Thu Mar 7 03:07:46 2024","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.91300000000000003,"15min":0.89500000000000002},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.2350000000000001,"5min":1.286,"15min":1.286},"last":1.0760000000000001},{"interface":"front","average":{"1min":0.86599999999999999,"5min":0.90000000000000002,"15min":0.91600000000000004},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":1.1379999999999999,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":1.01}]},{"osd":5,"last update":"Thu Mar 7 03:07:31 2024","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.77900000000000003,"15min":0.72399999999999998},"min":{"1min":0.40000000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.2949999999999999,"5min":1.349,"15min":1.349},"last":1.0449999999999999},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.79400000000000004,"15min":0.754},"min":{"1min":0.41099999999999998,"5min":0.374,"15min":0.374},"max":{"1min":1.2290000000000001,"5min":1.3,"15min":1.3},"last":0.97499999999999998}]},{"osd":6,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.874,"15min":0.85699999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.2729999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.93300000000000005,"15min":0.93300000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.3029999999999999,"5min":1.5,"15min":1.5},"last":0.75700000000000001}]},{"osd":7,"last update":"Thu Mar 7 03:08:09 2024","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.0489999999999999,"15min":1.0489999999999999},"min":{"1min":0.63,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.3440000000000001,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.95299999999999996},{"interface":"front","average":{"1min":0.998,"5min":0.96899999999999997,"15min":0.96899999999999997},"min":{"1min":0.57299999999999995,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.419,"5min":1.419,"15min":1.419},"last":1.1719999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724604,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:07:32 2024","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.85199999999999998,"15min":0.82699999999999996},"min":{"1min":0.64200000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.5189999999999999,"5min":1.665,"15min":1.665},"last":1.1719999999999999},{"interface":"front","average":{"1min":0.88900000000000001,"5min":0.85299999999999998,"15min":0.84299999999999997},"min":{"1min":0.59799999999999998,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.212,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":1.0800000000000001}]},{"osd":1,"last update":"Thu Mar 7 03:07:32 2024","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.89100000000000001,"15min":0.85999999999999999},"min":{"1min":0.626,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.204,"5min":1.276,"15min":1.276},"last":1.109},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.871,"15min":0.87},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.3089999999999999,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.98399999999999999}]},{"osd":2,"last update":"Thu Mar 7 03:07:32 2024","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.93000000000000005,"15min":0.93400000000000005},"min":{"1min":0.59199999999999997,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.337,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":1.143},{"interface":"front","average":{"1min":0.996,"5min":0.91400000000000003,"15min":0.88100000000000001},"min":{"1min":0.63900000000000001,"5min":0.438,"15min":0.438},"max":{"1min":1.252,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":1.016}]},{"osd":3,"last update":"Thu Mar 7 03:07:32 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.93100000000000005,"15min":0.91800000000000004},"min":{"1min":0.60299999999999998,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.419,"5min":1.419,"15min":1.419},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.95999999999999996,"15min":0.94199999999999995},"min":{"1min":0.68200000000000005,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.3440000000000001,"5min":1.593,"15min":1.593},"last":0.86499999999999999}]},{"osd":4,"last update":"Thu Mar 7 03:07:32 2024","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.81000000000000005,"15min":0.80600000000000005},"min":{"1min":0.504,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.4019999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.052,"5min":0.89000000000000001,"15min":0.84799999999999998},"min":{"1min":0.61099999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.6679999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":1.2270000000000001}]},{"osd":6,"last update":"Thu Mar 7 03:07:21 2024","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.95799999999999996,"15min":0.97199999999999998},"min":{"1min":0.54300000000000004,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.546,"5min":1.546,"15min":1.546},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.1020000000000001,"5min":0.97699999999999998,"15min":0.97899999999999998},"min":{"1min":0.56999999999999995,"5min":0.379,"15min":0.379},"max":{"1min":1.627,"5min":1.627,"15min":1.627},"last":0.69599999999999995}]},{"osd":7,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":0.97699999999999998,"15min":0.95099999999999996},"min":{"1min":0.66500000000000004,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.51,"5min":1.51,"15min":1.51},"last":0.90100000000000002},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.0529999999999999,"15min":1.0149999999999999},"min":{"1min":0.93700000000000006,"5min":0.435,"15min":0.435},"max":{"1min":1.496,"5min":1.782,"15min":1.782},"last":1.3089999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-07T03:08:21.966 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph pg dump --format=json 2024-03-07T03:08:22.815 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:08:22 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:08:22] "GET /metrics HTTP/1.1" 200 28562 "" "Prometheus/2.43.0" 2024-03-07T03:08:22.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:22 smithi145 bash[21339]: cluster 2024-03-07T03:08:20.394938+0000 mgr.y (mgr.24302) 103 : cluster 0 pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:22.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:22 smithi145 bash[21339]: audit 2024-03-07T03:08:21.182801+0000 mgr.y (mgr.24302) 104 : audit 0 from='client.14559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:08:22.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:22 smithi145 bash[30906]: cluster 2024-03-07T03:08:20.394938+0000 mgr.y (mgr.24302) 103 : cluster 0 pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:22.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:22 smithi145 bash[30906]: audit 2024-03-07T03:08:21.182801+0000 mgr.y (mgr.24302) 104 : audit 0 from='client.14559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:08:22.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:22 smithi189 bash[23614]: cluster 2024-03-07T03:08:20.394938+0000 mgr.y (mgr.24302) 103 : cluster 0 pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:22.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:22 smithi189 bash[23614]: audit 2024-03-07T03:08:21.182801+0000 mgr.y (mgr.24302) 104 : audit 0 from='client.14559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:08:24.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:24 smithi145 bash[21339]: cluster 2024-03-07T03:08:22.395520+0000 mgr.y (mgr.24302) 105 : cluster 0 pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:24.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:24 smithi145 bash[21339]: audit 2024-03-07T03:08:24.338090+0000 mon.c (mon.2) 72 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:24.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:24 smithi145 bash[30906]: cluster 2024-03-07T03:08:22.395520+0000 mgr.y (mgr.24302) 105 : cluster 0 pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:24.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:24 smithi145 bash[30906]: audit 2024-03-07T03:08:24.338090+0000 mon.c (mon.2) 72 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:24.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:24 smithi189 bash[23614]: cluster 2024-03-07T03:08:22.395520+0000 mgr.y (mgr.24302) 105 : cluster 0 pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:24.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:24 smithi189 bash[23614]: audit 2024-03-07T03:08:24.338090+0000 mon.c (mon.2) 72 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:26.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:25 smithi145 bash[30906]: cluster 2024-03-07T03:08:24.396131+0000 mgr.y (mgr.24302) 106 : cluster 0 pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:26.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:25 smithi145 bash[30906]: audit 2024-03-07T03:08:24.714982+0000 mon.a (mon.0) 759 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:26.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:25 smithi145 bash[30906]: audit 2024-03-07T03:08:24.723986+0000 mon.a (mon.0) 760 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:26.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:25 smithi145 bash[30906]: audit 2024-03-07T03:08:25.206072+0000 mon.c (mon.2) 73 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:26.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:25 smithi145 bash[30906]: audit 2024-03-07T03:08:25.207684+0000 mon.c (mon.2) 74 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:26.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:25 smithi145 bash[30906]: audit 2024-03-07T03:08:25.219935+0000 mon.a (mon.0) 761 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:26.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:25 smithi145 bash[21339]: cluster 2024-03-07T03:08:24.396131+0000 mgr.y (mgr.24302) 106 : cluster 0 pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:26.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:25 smithi145 bash[21339]: audit 2024-03-07T03:08:24.714982+0000 mon.a (mon.0) 759 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:26.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:25 smithi145 bash[21339]: audit 2024-03-07T03:08:24.723986+0000 mon.a (mon.0) 760 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:26.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:25 smithi145 bash[21339]: audit 2024-03-07T03:08:25.206072+0000 mon.c (mon.2) 73 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:26.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:25 smithi145 bash[21339]: audit 2024-03-07T03:08:25.207684+0000 mon.c (mon.2) 74 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:26.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:25 smithi145 bash[21339]: audit 2024-03-07T03:08:25.219935+0000 mon.a (mon.0) 761 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:26.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:25 smithi189 bash[23614]: cluster 2024-03-07T03:08:24.396131+0000 mgr.y (mgr.24302) 106 : cluster 0 pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:26.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:25 smithi189 bash[23614]: audit 2024-03-07T03:08:24.714982+0000 mon.a (mon.0) 759 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:26.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:25 smithi189 bash[23614]: audit 2024-03-07T03:08:24.723986+0000 mon.a (mon.0) 760 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:26.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:25 smithi189 bash[23614]: audit 2024-03-07T03:08:25.206072+0000 mon.c (mon.2) 73 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:26.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:25 smithi189 bash[23614]: audit 2024-03-07T03:08:25.207684+0000 mon.c (mon.2) 74 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:26.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:25 smithi189 bash[23614]: audit 2024-03-07T03:08:25.219935+0000 mon.a (mon.0) 761 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:26.698 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:08:28.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:27 smithi145 bash[21339]: cluster 2024-03-07T03:08:26.396703+0000 mgr.y (mgr.24302) 107 : cluster 0 pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:28.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:27 smithi145 bash[30906]: cluster 2024-03-07T03:08:26.396703+0000 mgr.y (mgr.24302) 107 : cluster 0 pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:28.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:27 smithi189 bash[23614]: cluster 2024-03-07T03:08:26.396703+0000 mgr.y (mgr.24302) 107 : cluster 0 pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:29.954 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:08:29.956 INFO:teuthology.orchestra.run.smithi145.stderr:dumped all 2024-03-07T03:08:30.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:30 smithi145 bash[21339]: cluster 2024-03-07T03:08:28.397262+0000 mgr.y (mgr.24302) 108 : cluster 0 pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:30.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:30 smithi145 bash[30906]: cluster 2024-03-07T03:08:28.397262+0000 mgr.y (mgr.24302) 108 : cluster 0 pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:30.752 INFO:teuthology.orchestra.run.smithi145.stdout:{"pg_ready":true,"pg_map":{"version":80,"stamp":"2024-03-07T03:08:28.396991+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":222896,"kb_used_data":6736,"kb_used_omap":13,"kb_used_meta":216050,"kb_avail":749738320,"statfs":{"total":767960285184,"available":767732039680,"internally_reserved":0,"allocated":6897664,"data_stored":3903531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14248,"internal_metadata":221235288},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003500"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:44.348056+0000","last_change":"2024-03-07T03:06:42.667758+0000","last_active":"2024-03-07T03:06:44.348056+0000","last_peered":"2024-03-07T03:06:44.348056+0000","last_clean":"2024-03-07T03:06:44.348056+0000","last_became_active":"2024-03-07T03:06:42.667171+0000","last_became_peered":"2024-03-07T03:06:42.667171+0000","last_unstale":"2024-03-07T03:06:44.348056+0000","last_undegraded":"2024-03-07T03:06:44.348056+0000","last_fullsized":"2024-03-07T03:06:44.348056+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:55:07.432703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:43.669050+0000","last_change":"2024-03-07T03:06:42.794071+0000","last_active":"2024-03-07T03:06:43.669050+0000","last_peered":"2024-03-07T03:06:43.669050+0000","last_clean":"2024-03-07T03:06:43.669050+0000","last_became_active":"2024-03-07T03:06:42.792899+0000","last_became_peered":"2024-03-07T03:06:42.792899+0000","last_unstale":"2024-03-07T03:06:43.669050+0000","last_undegraded":"2024-03-07T03:06:43.669050+0000","last_fullsized":"2024-03-07T03:06:43.669050+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:07:21.918130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:43.670025+0000","last_change":"2024-03-07T03:06:42.792519+0000","last_active":"2024-03-07T03:06:43.670025+0000","last_peered":"2024-03-07T03:06:43.670025+0000","last_clean":"2024-03-07T03:06:43.670025+0000","last_became_active":"2024-03-07T03:06:42.792136+0000","last_became_peered":"2024-03-07T03:06:42.792136+0000","last_unstale":"2024-03-07T03:06:43.670025+0000","last_undegraded":"2024-03-07T03:06:43.670025+0000","last_fullsized":"2024-03-07T03:06:43.670025+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:50:03.686748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:43.669592+0000","last_change":"2024-03-07T03:06:42.668416+0000","last_active":"2024-03-07T03:06:43.669592+0000","last_peered":"2024-03-07T03:06:43.669592+0000","last_clean":"2024-03-07T03:06:43.669592+0000","last_became_active":"2024-03-07T03:06:42.668043+0000","last_became_peered":"2024-03-07T03:06:42.668043+0000","last_unstale":"2024-03-07T03:06:43.669592+0000","last_undegraded":"2024-03-07T03:06:43.669592+0000","last_fullsized":"2024-03-07T03:06:43.669592+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:07:59.168239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":260,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-03-07T03:06:52.849844+0000","last_change":"2024-03-07T03:05:08.667040+0000","last_active":"2024-03-07T03:06:52.849844+0000","last_peered":"2024-03-07T03:06:52.849844+0000","last_clean":"2024-03-07T03:06:52.849844+0000","last_became_active":"2024-03-07T03:05:08.665926+0000","last_became_peered":"2024-03-07T03:05:08.665926+0000","last_unstale":"2024-03-07T03:06:52.849844+0000","last_undegraded":"2024-03-07T03:06:52.849844+0000","last_fullsized":"2024-03-07T03:06:52.849844+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:01:00.212577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:01:00.212577+0000","last_clean_scrub_stamp":"2024-03-07T03:01:00.212577+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:11:34.307802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:44.849273+0000","last_change":"2024-03-07T03:06:42.670236+0000","last_active":"2024-03-07T03:06:44.849273+0000","last_peered":"2024-03-07T03:06:44.849273+0000","last_clean":"2024-03-07T03:06:44.849273+0000","last_became_active":"2024-03-07T03:06:42.669808+0000","last_became_peered":"2024-03-07T03:06:42.669808+0000","last_unstale":"2024-03-07T03:06:44.849273+0000","last_undegraded":"2024-03-07T03:06:44.849273+0000","last_fullsized":"2024-03-07T03:06:44.849273+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:42:36.928386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:43.899381+0000","last_change":"2024-03-07T03:06:42.795963+0000","last_active":"2024-03-07T03:06:43.899381+0000","last_peered":"2024-03-07T03:06:43.899381+0000","last_clean":"2024-03-07T03:06:43.899381+0000","last_became_active":"2024-03-07T03:06:42.795033+0000","last_became_peered":"2024-03-07T03:06:42.795033+0000","last_unstale":"2024-03-07T03:06:43.899381+0000","last_undegraded":"2024-03-07T03:06:43.899381+0000","last_fullsized":"2024-03-07T03:06:43.899381+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:12:44.845051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:44.849204+0000","last_change":"2024-03-07T03:06:42.667215+0000","last_active":"2024-03-07T03:06:44.849204+0000","last_peered":"2024-03-07T03:06:44.849204+0000","last_clean":"2024-03-07T03:06:44.849204+0000","last_became_active":"2024-03-07T03:06:42.666675+0000","last_became_peered":"2024-03-07T03:06:42.666675+0000","last_unstale":"2024-03-07T03:06:44.849204+0000","last_undegraded":"2024-03-07T03:06:44.849204+0000","last_fullsized":"2024-03-07T03:06:44.849204+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:35:31.916960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:06:43.669078+0000","last_change":"2024-03-07T03:06:42.666671+0000","last_active":"2024-03-07T03:06:43.669078+0000","last_peered":"2024-03-07T03:06:43.669078+0000","last_clean":"2024-03-07T03:06:43.669078+0000","last_became_active":"2024-03-07T03:06:42.666402+0000","last_became_peered":"2024-03-07T03:06:42.666402+0000","last_unstale":"2024-03-07T03:06:43.669078+0000","last_undegraded":"2024-03-07T03:06:43.669078+0000","last_fullsized":"2024-03-07T03:06:43.669078+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:28:33.638276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201322,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28224,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716928,"statfs":{"total":95995035648,"available":95966134272,"internally_reserved":0,"allocated":1232896,"data_stored":856851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.93600000000000005,"15min":0.92400000000000004},"min":{"1min":0.60199999999999998,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.1910000000000001,"5min":1.524,"15min":1.524},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1,"15min":1.03},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.0880000000000001}]},{"osd":1,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.92000000000000004,"15min":0.93000000000000005},"min":{"1min":0.59499999999999997,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.2430000000000001,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.997},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.0349999999999999,"15min":1.0580000000000001},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.3,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.91400000000000003}]},{"osd":2,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.016,"15min":1.048},"min":{"1min":0.72899999999999998,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.1459999999999999,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.98799999999999999,"15min":0.98599999999999999},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.1990000000000001,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":0.875}]},{"osd":3,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.92900000000000005,"15min":0.94399999999999995},"min":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.327,"5min":1.327,"15min":1.327},"last":1.1439999999999999},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.0569999999999999,"15min":1.079},"min":{"1min":0.68700000000000006,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.2450000000000001,"5min":1.403,"15min":1.403},"last":1.173}]},{"osd":4,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.93000000000000005,"15min":0.92000000000000004},"min":{"1min":0.70499999999999996,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.2709999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.032,"5min":0.98499999999999999,"15min":0.98899999999999999},"min":{"1min":0.73299999999999998,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.234}]},{"osd":5,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.98899999999999999,"15min":0.98499999999999999},"min":{"1min":0.79000000000000004,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.2270000000000001,"5min":1.446,"15min":1.446},"last":1.05},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.0920000000000001,"15min":1.1140000000000001},"min":{"1min":0.75800000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":1.202}]},{"osd":6,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.125,"15min":1.135},"min":{"1min":0.86499999999999999,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3600000000000001,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":1.3},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.1220000000000001,"15min":1.1200000000000001},"min":{"1min":0.94199999999999995,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.327,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":1.272}]}]},{"osd":6,"up_from":47,"seq":201863462964,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28220,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716932,"statfs":{"total":95995035648,"available":95966138368,"internally_reserved":0,"allocated":1228800,"data_stored":856578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.89300000000000002,"15min":0.92300000000000004},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.349,"5min":1.349,"15min":1.349},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.91100000000000003,"15min":0.91300000000000003},"min":{"1min":0.60899999999999999,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.496,"5min":1.496,"15min":1.496},"last":0.92800000000000005}]},{"osd":1,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.91500000000000004,"15min":0.90800000000000003},"min":{"1min":0.64900000000000002,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.4510000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.1699999999999999},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.91400000000000003,"15min":0.92600000000000005},"min":{"1min":0.47999999999999998,"5min":0.438,"15min":0.438},"max":{"1min":1.454,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":0.53400000000000003}]},{"osd":2,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.91500000000000004,"15min":0.90500000000000003},"min":{"1min":0.60799999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.3420000000000001,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.79100000000000004},{"interface":"front","average":{"1min":1.1870000000000001,"5min":0.98399999999999999,"15min":0.96699999999999997},"min":{"1min":0.63100000000000001,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":3.6360000000000001,"5min":3.6360000000000001,"15min":3.6360000000000001},"last":0.752}]},{"osd":3,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":0.96699999999999997,"15min":0.94899999999999995},"min":{"1min":0.70999999999999996,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":3.9169999999999998,"5min":3.9169999999999998,"15min":3.9169999999999998},"last":1.0880000000000001},{"interface":"front","average":{"1min":1.21,"5min":1.02,"15min":1.004},"min":{"1min":0.64500000000000002,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":3.4790000000000001,"5min":3.4790000000000001,"15min":3.4790000000000001},"last":0.877}]},{"osd":4,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.86199999999999999,"15min":0.81999999999999995},"min":{"1min":0.58399999999999996,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.0339999999999998,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.84899999999999998},{"interface":"front","average":{"1min":1.103,"5min":0.91300000000000003,"15min":0.89100000000000001},"min":{"1min":0.55900000000000005,"5min":0.307,"15min":0.307},"max":{"1min":1.7250000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.226}]},{"osd":5,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":0.94699999999999995,"15min":0.90500000000000003},"min":{"1min":0.746,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.1720000000000002,"5min":2.1720000000000002,"15min":2.1720000000000002},"last":1.0589999999999999},{"interface":"front","average":{"1min":1.121,"5min":0.98299999999999998,"15min":0.96499999999999997},"min":{"1min":0.51900000000000002,"5min":0.374,"15min":0.374},"max":{"1min":1.587,"5min":1.587,"15min":1.587},"last":1.27}]},{"osd":7,"last update":"Thu Mar 7 03:08:14 2024","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.042,"15min":1.022},"min":{"1min":0.84299999999999997,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.3069999999999999,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.3109999999999999},{"interface":"front","average":{"1min":1.105,"5min":1.1180000000000001,"15min":1.0900000000000001},"min":{"1min":0.85599999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.3100000000000001,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":1.0189999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574950,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:07:20 2024","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.67100000000000004,"15min":0.58199999999999996},"min":{"1min":0.33200000000000002,"5min":0.246,"15min":0.246},"max":{"1min":1.0349999999999999,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.76800000000000002,"15min":0.60599999999999998},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.28899999999999998},"max":{"1min":1.2010000000000001,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":1.141}]},{"osd":2,"last update":"Thu Mar 7 03:08:11 2024","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.76800000000000002,"15min":0.66400000000000003},"min":{"1min":0.36899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.01,"5min":1.3169999999999999,"15min":1.3169999999999999},"last":0.93500000000000005},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.76300000000000001,"15min":0.64800000000000002},"min":{"1min":0.40500000000000003,"5min":0.308,"15min":0.308},"max":{"1min":1.0469999999999999,"5min":1.327,"15min":1.327},"last":0.69699999999999995}]},{"osd":3,"last update":"Thu Mar 7 03:08:08 2024","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.83899999999999997,"15min":0.81100000000000005},"min":{"1min":0.40500000000000003,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":1.282},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.78900000000000003,"15min":0.78100000000000003},"min":{"1min":0.36899999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.0640000000000001,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.98299999999999998}]},{"osd":4,"last update":"Thu Mar 7 03:07:49 2024","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.94099999999999995,"15min":0.93100000000000005},"min":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.3109999999999999,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.0580000000000001},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.97999999999999998,"15min":0.92400000000000004},"min":{"1min":0.53800000000000003,"5min":0.503,"15min":0.503},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":1.175}]},{"osd":5,"last update":"Thu Mar 7 03:07:33 2024","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.97599999999999998,"15min":0.95799999999999996},"min":{"1min":0.65700000000000003,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.427,"5min":1.427,"15min":1.427},"last":1.0940000000000001},{"interface":"front","average":{"1min":1.036,"5min":1.0549999999999999,"15min":1.0429999999999999},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.4470000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.2450000000000001}]},{"osd":6,"last update":"Thu Mar 7 03:07:22 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.089,"15min":1.089},"min":{"1min":0.61499999999999999,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.5109999999999999,"5min":1.8520000000000001,"15min":1.8520000000000001},"last":1.02},{"interface":"front","average":{"1min":1.081,"5min":1.101,"15min":1.1120000000000001},"min":{"1min":0.67300000000000004,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.4990000000000001,"5min":1.512,"15min":1.512},"last":1.212}]},{"osd":7,"last update":"Thu Mar 7 03:08:11 2024","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.145,"15min":1.177},"min":{"1min":0.67400000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.401,"5min":1.619,"15min":1.619},"last":1.3340000000000001},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.133,"15min":1.135},"min":{"1min":0.71299999999999997,"5min":0.622,"15min":0.622},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":1.371}]}]},{"osd":0,"up_from":8,"seq":34359738480,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28228,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716924,"statfs":{"total":95995035648,"available":95966130176,"internally_reserved":0,"allocated":1236992,"data_stored":857182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Mar 7 03:07:24 2024","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.71299999999999997,"15min":0.58499999999999996},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.25},"max":{"1min":1.3340000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.77900000000000003,"15min":0.63600000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.25900000000000001},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.5}]},{"osd":2,"last update":"Thu Mar 7 03:08:03 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.77900000000000003,"15min":0.67700000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.28499999999999998},"max":{"1min":0.94199999999999995,"5min":1.248,"15min":1.2869999999999999},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.81799999999999995,"15min":0.71699999999999997},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":1.177,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.78800000000000003}]},{"osd":3,"last update":"Thu Mar 7 03:08:03 2024","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.873,"15min":0.80900000000000005},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.175,"5min":1.6180000000000001,"15min":1.6180000000000001},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.85899999999999999,"5min":0.89300000000000002,"15min":0.82099999999999995},"min":{"1min":0.32200000000000001,"5min":0.316,"15min":0.316},"max":{"1min":1.1970000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.86199999999999999}]},{"osd":4,"last update":"Thu Mar 7 03:07:52 2024","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.97899999999999998,"15min":0.95599999999999996},"min":{"1min":0.64400000000000002,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.3680000000000001,"5min":1.375,"15min":1.375},"last":1.1339999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.91600000000000004,"15min":0.91000000000000003},"min":{"1min":0.59799999999999998,"5min":0.498,"15min":0.498},"max":{"1min":1.2649999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.997}]},{"osd":5,"last update":"Thu Mar 7 03:07:33 2024","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.0349999999999999,"15min":0.98999999999999999},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.8109999999999999,"5min":1.8109999999999999,"15min":1.8109999999999999},"last":1.24},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.0369999999999999,"15min":1.0009999999999999},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.6579999999999999,"5min":1.6579999999999999,"15min":1.6579999999999999},"last":0.97299999999999998}]},{"osd":6,"last update":"Thu Mar 7 03:08:21 2024","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.008,"15min":1.018},"min":{"1min":0.72799999999999998,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.1699999999999999,"15min":1.1919999999999999},"min":{"1min":0.77600000000000002,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.3819999999999999,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.0920000000000001}]},{"osd":7,"last update":"Thu Mar 7 03:08:13 2024","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.0489999999999999,"15min":1.0609999999999999},"min":{"1min":0.80300000000000005,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.319,"5min":1.319,"15min":1.319},"last":1.0660000000000001},{"interface":"front","average":{"1min":1.135,"5min":1.1759999999999999,"15min":1.1919999999999999},"min":{"1min":0.70299999999999996,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.724,"5min":1.724,"15min":1.724},"last":1.1879999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411420,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:08:12 2024","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.69999999999999996,"15min":0.61899999999999999},"min":{"1min":0.35099999999999998,"5min":0.318,"15min":0.307},"max":{"1min":1.153,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.873},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.76100000000000001,"15min":0.66000000000000003},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.292,"5min":1.292,"15min":1.292},"last":0.94699999999999995}]},{"osd":1,"last update":"Thu Mar 7 03:08:12 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.72899999999999998,"15min":0.67100000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":0.877,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":1.095},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.77800000000000002,"15min":0.70199999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.28999999999999998},"max":{"1min":1.0049999999999999,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.71199999999999997}]},{"osd":3,"last update":"Thu Mar 7 03:08:12 2024","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.83199999999999996,"15min":0.80700000000000005},"min":{"1min":0.38100000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.042,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":1.018},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.82299999999999995,"15min":0.80000000000000004},"min":{"1min":0.41499999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.1799999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.76500000000000001}]},{"osd":4,"last update":"Thu Mar 7 03:07:45 2024","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":0.97999999999999998,"15min":0.95399999999999996},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":1.0700000000000001},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.95499999999999996,"15min":0.92400000000000004},"min":{"1min":0.57799999999999996,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.649,"5min":1.649,"15min":1.649},"last":0.84399999999999997}]},{"osd":5,"last update":"Thu Mar 7 03:07:37 2024","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.98399999999999999,"15min":0.97099999999999997},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.1699999999999999,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.1719999999999999},{"interface":"front","average":{"1min":1.0249999999999999,"5min":0.98799999999999999,"15min":0.98499999999999999},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.53,"5min":1.53,"15min":1.53},"last":0.99199999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:07:22 2024","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.056,"15min":1.075},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.599,"5min":1.599,"15min":1.599},"last":1.198},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.1120000000000001,"15min":1.1180000000000001},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.746,"5min":1.746,"15min":1.746},"last":1.0509999999999999}]},{"osd":7,"last update":"Thu Mar 7 03:08:12 2024","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.0980000000000001,"15min":1.095},"min":{"1min":0.753,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.407,"5min":1.6759999999999999,"15min":1.6759999999999999},"last":1.129},{"interface":"front","average":{"1min":1.111,"5min":1.1559999999999999,"15min":1.155},"min":{"1min":0.56499999999999995,"5min":0.501,"15min":0.501},"max":{"1min":1.3899999999999999,"5min":1.696,"15min":1.696},"last":1.149}]}]},{"osd":3,"up_from":26,"seq":111669149776,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:08:06 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.66000000000000003,"15min":0.67900000000000005},"min":{"1min":0.29599999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.1759999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.753,"15min":0.73999999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.89700000000000002,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.64900000000000002}]},{"osd":1,"last update":"Thu Mar 7 03:08:06 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.77100000000000002,"15min":0.78500000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.165,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.625,"5min":0.746,"15min":0.748},"min":{"1min":0.35299999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.97399999999999998,"5min":1.476,"15min":1.476},"last":0.72999999999999998}]},{"osd":2,"last update":"Thu Mar 7 03:08:06 2024","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.80100000000000005,"15min":0.81200000000000006},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.089,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":1.0609999999999999},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.79500000000000004,"15min":0.76800000000000002},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.96699999999999997,"5min":1.407,"15min":1.407},"last":0.79600000000000004}]},{"osd":4,"last update":"Thu Mar 7 03:07:49 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.95199999999999996,"15min":0.89500000000000002},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.329,"5min":1.361,"15min":1.361},"last":1.0900000000000001},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.97199999999999998,"15min":0.94399999999999995},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.4330000000000001,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":1.1040000000000001}]},{"osd":5,"last update":"Thu Mar 7 03:07:37 2024","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.96099999999999997,"15min":0.96499999999999997},"min":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.224,"5min":1.468,"15min":1.468},"last":0.91700000000000004},{"interface":"front","average":{"1min":0.97799999999999998,"5min":1.0529999999999999,"15min":1.081},"min":{"1min":0.61799999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.3360000000000001,"5min":1.492,"15min":1.492},"last":1.24}]},{"osd":6,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.0620000000000001,"15min":1.0720000000000001},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.431,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":1.02},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.0569999999999999,"15min":1.0620000000000001},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.5589999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":1.1499999999999999}]},{"osd":7,"last update":"Thu Mar 7 03:08:06 2024","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.0720000000000001,"15min":1.1000000000000001},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.3080000000000001,"5min":1.45,"15min":1.45},"last":1.04},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.1040000000000001,"15min":1.1220000000000001},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.2929999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.1890000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986247,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:07:46 2024","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.84299999999999997,"15min":0.85899999999999999},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.198,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.87},{"interface":"front","average":{"1min":0.871,"5min":0.871,"15min":0.85499999999999998},"min":{"1min":0.54000000000000004,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.226,"5min":1.323,"15min":1.323},"last":0.89400000000000002}]},{"osd":1,"last update":"Thu Mar 7 03:07:46 2024","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.86599999999999999,"15min":0.86799999999999999},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.1919999999999999,"5min":1.194,"15min":1.194},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.88600000000000001,"5min":0.89200000000000002,"15min":0.88900000000000001},"min":{"1min":0.52700000000000002,"5min":0.498,"15min":0.498},"max":{"1min":1.27,"5min":1.345,"15min":1.345},"last":1.208}]},{"osd":2,"last update":"Thu Mar 7 03:07:46 2024","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.92300000000000004,"15min":0.90500000000000003},"min":{"1min":0.59099999999999997,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.264,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.871,"5min":0.88700000000000001,"15min":0.85499999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.1220000000000001,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":0.83999999999999997}]},{"osd":3,"last update":"Thu Mar 7 03:07:46 2024","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.91300000000000003,"15min":0.89500000000000002},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.2350000000000001,"5min":1.286,"15min":1.286},"last":0.93300000000000005},{"interface":"front","average":{"1min":0.86599999999999999,"5min":0.90000000000000002,"15min":0.91600000000000004},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":1.1379999999999999,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.73599999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:07:31 2024","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.77900000000000003,"15min":0.72399999999999998},"min":{"1min":0.40000000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.2949999999999999,"5min":1.349,"15min":1.349},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.79400000000000004,"15min":0.754},"min":{"1min":0.41099999999999998,"5min":0.374,"15min":0.374},"max":{"1min":1.2290000000000001,"5min":1.3,"15min":1.3},"last":1.145}]},{"osd":6,"last update":"Thu Mar 7 03:07:23 2024","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.874,"15min":0.85699999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.2729999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":1.04},{"interface":"front","average":{"1min":0.92000000000000004,"5min":0.93300000000000005,"15min":0.93300000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.3029999999999999,"5min":1.5,"15min":1.5},"last":1.1060000000000001}]},{"osd":7,"last update":"Thu Mar 7 03:08:09 2024","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.0489999999999999,"15min":1.0489999999999999},"min":{"1min":0.63,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.3440000000000001,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":1.167},{"interface":"front","average":{"1min":0.998,"5min":0.96899999999999997,"15min":0.96899999999999997},"min":{"1min":0.57299999999999995,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.419,"5min":1.419,"15min":1.419},"last":0.97399999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724605,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:07:32 2024","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.85199999999999998,"15min":0.82699999999999996},"min":{"1min":0.64200000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.5189999999999999,"5min":1.665,"15min":1.665},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.88900000000000001,"5min":0.85299999999999998,"15min":0.84299999999999997},"min":{"1min":0.59799999999999998,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.212,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":1.218}]},{"osd":1,"last update":"Thu Mar 7 03:07:32 2024","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.89100000000000001,"15min":0.85999999999999999},"min":{"1min":0.626,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.204,"5min":1.276,"15min":1.276},"last":1.258},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.871,"15min":0.87},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.3089999999999999,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":1.1040000000000001}]},{"osd":2,"last update":"Thu Mar 7 03:07:32 2024","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.93000000000000005,"15min":0.93400000000000005},"min":{"1min":0.59199999999999997,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.337,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.999},{"interface":"front","average":{"1min":0.996,"5min":0.91400000000000003,"15min":0.88100000000000001},"min":{"1min":0.63900000000000001,"5min":0.438,"15min":0.438},"max":{"1min":1.252,"5min":1.4370000000000001,"15min":1.4370000000000001},"last":1.0489999999999999}]},{"osd":3,"last update":"Thu Mar 7 03:07:32 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.93100000000000005,"15min":0.91800000000000004},"min":{"1min":0.60299999999999998,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.419,"5min":1.419,"15min":1.419},"last":0.91900000000000004},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.95999999999999996,"15min":0.94199999999999995},"min":{"1min":0.68200000000000005,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.3440000000000001,"5min":1.593,"15min":1.593},"last":0.96799999999999997}]},{"osd":4,"last update":"Thu Mar 7 03:07:32 2024","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.81000000000000005,"15min":0.80600000000000005},"min":{"1min":0.504,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.4019999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":1.3129999999999999},{"interface":"front","average":{"1min":1.052,"5min":0.89000000000000001,"15min":0.84799999999999998},"min":{"1min":0.61099999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.6679999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":1.073}]},{"osd":6,"last update":"Thu Mar 7 03:07:21 2024","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":0.95799999999999996,"15min":0.97199999999999998},"min":{"1min":0.54300000000000004,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.546,"5min":1.546,"15min":1.546},"last":1.198},{"interface":"front","average":{"1min":1.1020000000000001,"5min":0.97699999999999998,"15min":0.97899999999999998},"min":{"1min":0.56999999999999995,"5min":0.379,"15min":0.379},"max":{"1min":1.627,"5min":1.627,"15min":1.627},"last":1.387}]},{"osd":7,"last update":"Thu Mar 7 03:08:10 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":0.97699999999999998,"15min":0.95099999999999996},"min":{"1min":0.66500000000000004,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.51,"5min":1.51,"15min":1.51},"last":1.365},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.0529999999999999,"15min":1.0149999999999999},"min":{"1min":0.93700000000000006,"5min":0.435,"15min":0.435},"max":{"1min":1.496,"5min":1.782,"15min":1.782},"last":1.133}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-07T03:08:30.754 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-07T03:08:30.754 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-07T03:08:30.754 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-07T03:08:30.754 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:08:30.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:30 smithi189 bash[23614]: cluster 2024-03-07T03:08:28.397262+0000 mgr.y (mgr.24302) 108 : cluster 0 pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:31.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:31 smithi145 bash[21339]: audit 2024-03-07T03:08:29.951581+0000 mgr.y (mgr.24302) 109 : audit 0 from='client.24482 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:08:31.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:31 smithi145 bash[30906]: audit 2024-03-07T03:08:29.951581+0000 mgr.y (mgr.24302) 109 : audit 0 from='client.24482 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:08:31.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:31 smithi189 bash[23614]: audit 2024-03-07T03:08:29.951581+0000 mgr.y (mgr.24302) 109 : audit 0 from='client.24482 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:08:32.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:32 smithi145 bash[21339]: cluster 2024-03-07T03:08:30.397860+0000 mgr.y (mgr.24302) 110 : cluster 0 pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:32.815 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:08:32 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:08:32] "GET /metrics HTTP/1.1" 200 28562 "" "Prometheus/2.43.0" 2024-03-07T03:08:32.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:32 smithi145 bash[30906]: cluster 2024-03-07T03:08:30.397860+0000 mgr.y (mgr.24302) 110 : cluster 0 pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:32.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:32 smithi189 bash[23614]: cluster 2024-03-07T03:08:30.397860+0000 mgr.y (mgr.24302) 110 : cluster 0 pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:33.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:33 smithi145 bash[21339]: audit 2024-03-07T03:08:32.448831+0000 mon.c (mon.2) 75 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:33.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:33 smithi145 bash[30906]: audit 2024-03-07T03:08:32.448831+0000 mon.c (mon.2) 75 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:33.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:33 smithi189 bash[23614]: audit 2024-03-07T03:08:32.448831+0000 mon.c (mon.2) 75 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:34.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:34 smithi145 bash[21339]: cluster 2024-03-07T03:08:32.398434+0000 mgr.y (mgr.24302) 111 : cluster 0 pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:34.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:34 smithi145 bash[30906]: cluster 2024-03-07T03:08:32.398434+0000 mgr.y (mgr.24302) 111 : cluster 0 pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:34.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:34 smithi189 bash[23614]: cluster 2024-03-07T03:08:32.398434+0000 mgr.y (mgr.24302) 111 : cluster 0 pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:35.481 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:08:36.671 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:36 smithi145 bash[21339]: cluster 2024-03-07T03:08:34.399136+0000 mgr.y (mgr.24302) 112 : cluster 0 pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:36.672 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:36 smithi145 bash[30906]: cluster 2024-03-07T03:08:34.399136+0000 mgr.y (mgr.24302) 112 : cluster 0 pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:36.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:36 smithi189 bash[23614]: cluster 2024-03-07T03:08:34.399136+0000 mgr.y (mgr.24302) 112 : cluster 0 pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:38.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:38 smithi145 bash[21339]: cluster 2024-03-07T03:08:36.399695+0000 mgr.y (mgr.24302) 113 : cluster 0 pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:38.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:38 smithi145 bash[30906]: cluster 2024-03-07T03:08:36.399695+0000 mgr.y (mgr.24302) 113 : cluster 0 pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:38.821 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:08:38.821 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-07T03:08:38.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:38 smithi189 bash[23614]: cluster 2024-03-07T03:08:36.399695+0000 mgr.y (mgr.24302) 113 : cluster 0 pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:39.672 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:39 smithi145 bash[21339]: audit 2024-03-07T03:08:38.828361+0000 mon.a (mon.0) 762 : audit 0 from='client.? 172.21.15.145:0/3243823692' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:08:39.672 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:39 smithi145 bash[30906]: audit 2024-03-07T03:08:38.828361+0000 mon.a (mon.0) 762 : audit 0 from='client.? 172.21.15.145:0/3243823692' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:08:39.673 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-07T03:08:39.674 INFO:tasks.cephadm:Setup complete, yielding 2024-03-07T03:08:39.674 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-07T03:08:39.683 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi145.front.sepia.ceph.com 2024-03-07T03:08:39.683 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- bash -c 'ceph fs volume create foo' 2024-03-07T03:08:39.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:39 smithi189 bash[23614]: audit 2024-03-07T03:08:38.828361+0000 mon.a (mon.0) 762 : audit 0 from='client.? 172.21.15.145:0/3243823692' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:08:40.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:40 smithi145 bash[21339]: cluster 2024-03-07T03:08:38.400282+0000 mgr.y (mgr.24302) 114 : cluster 0 pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:40.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:40 smithi145 bash[30906]: cluster 2024-03-07T03:08:38.400282+0000 mgr.y (mgr.24302) 114 : cluster 0 pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:40.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:40 smithi189 bash[23614]: cluster 2024-03-07T03:08:38.400282+0000 mgr.y (mgr.24302) 114 : cluster 0 pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:41.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:41 smithi145 bash[21339]: cluster 2024-03-07T03:08:40.400922+0000 mgr.y (mgr.24302) 115 : cluster 0 pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:41.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:41 smithi145 bash[30906]: cluster 2024-03-07T03:08:40.400922+0000 mgr.y (mgr.24302) 115 : cluster 0 pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:41.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:41 smithi189 bash[23614]: cluster 2024-03-07T03:08:40.400922+0000 mgr.y (mgr.24302) 115 : cluster 0 pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:42.814 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:08:42 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:08:42] "GET /metrics HTTP/1.1" 200 28564 "" "Prometheus/2.43.0" 2024-03-07T03:08:44.413 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:08:44.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:44 smithi145 bash[30906]: cluster 2024-03-07T03:08:42.401298+0000 mgr.y (mgr.24302) 116 : cluster 0 pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:44.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:44 smithi145 bash[21339]: cluster 2024-03-07T03:08:42.401298+0000 mgr.y (mgr.24302) 116 : cluster 0 pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:44.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:44 smithi189 bash[23614]: cluster 2024-03-07T03:08:42.401298+0000 mgr.y (mgr.24302) 116 : cluster 0 pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:46.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:46 smithi145 bash[21339]: cluster 2024-03-07T03:08:44.401910+0000 mgr.y (mgr.24302) 117 : cluster 0 pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:46.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:46 smithi145 bash[30906]: cluster 2024-03-07T03:08:44.401910+0000 mgr.y (mgr.24302) 117 : cluster 0 pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:46.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:46 smithi189 bash[23614]: cluster 2024-03-07T03:08:44.401910+0000 mgr.y (mgr.24302) 117 : cluster 0 pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:48.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:48 smithi145 bash[21339]: cluster 2024-03-07T03:08:46.402518+0000 mgr.y (mgr.24302) 118 : cluster 0 pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:48.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:48 smithi145 bash[21339]: audit 2024-03-07T03:08:47.449276+0000 mon.c (mon.2) 76 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:48.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:48 smithi145 bash[21339]: audit 2024-03-07T03:08:47.714848+0000 mon.c (mon.2) 77 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-07T03:08:48.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:48 smithi145 bash[21339]: audit 2024-03-07T03:08:47.715548+0000 mon.a (mon.0) 763 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-07T03:08:48.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:48 smithi145 bash[30906]: cluster 2024-03-07T03:08:46.402518+0000 mgr.y (mgr.24302) 118 : cluster 0 pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:48.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:48 smithi145 bash[30906]: audit 2024-03-07T03:08:47.449276+0000 mon.c (mon.2) 76 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:48.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:48 smithi145 bash[30906]: audit 2024-03-07T03:08:47.714848+0000 mon.c (mon.2) 77 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-07T03:08:48.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:48 smithi145 bash[30906]: audit 2024-03-07T03:08:47.715548+0000 mon.a (mon.0) 763 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-07T03:08:48.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:48 smithi189 bash[23614]: cluster 2024-03-07T03:08:46.402518+0000 mgr.y (mgr.24302) 118 : cluster 0 pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:48.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:48 smithi189 bash[23614]: audit 2024-03-07T03:08:47.449276+0000 mon.c (mon.2) 76 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:48.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:48 smithi189 bash[23614]: audit 2024-03-07T03:08:47.714848+0000 mon.c (mon.2) 77 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-07T03:08:48.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:48 smithi189 bash[23614]: audit 2024-03-07T03:08:47.715548+0000 mon.a (mon.0) 763 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-07T03:08:49.715 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:49 smithi145 bash[30906]: audit 2024-03-07T03:08:47.714133+0000 mgr.y (mgr.24302) 119 : audit 0 from='client.24494 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:49.715 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:49 smithi145 bash[30906]: audit 2024-03-07T03:08:48.377681+0000 mon.a (mon.0) 764 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-07T03:08:49.715 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:49 smithi145 bash[30906]: cluster 2024-03-07T03:08:48.377874+0000 mon.a (mon.0) 765 : cluster 0 osdmap e65: 8 total, 8 up, 8 in 2024-03-07T03:08:49.715 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:49 smithi145 bash[30906]: audit 2024-03-07T03:08:48.388505+0000 mon.c (mon.2) 78 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-07T03:08:49.715 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:49 smithi145 bash[30906]: audit 2024-03-07T03:08:48.389895+0000 mon.a (mon.0) 766 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-07T03:08:49.715 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:49 smithi145 bash[21339]: audit 2024-03-07T03:08:47.714133+0000 mgr.y (mgr.24302) 119 : audit 0 from='client.24494 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:49.716 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:49 smithi145 bash[21339]: audit 2024-03-07T03:08:48.377681+0000 mon.a (mon.0) 764 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-07T03:08:49.716 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:49 smithi145 bash[21339]: cluster 2024-03-07T03:08:48.377874+0000 mon.a (mon.0) 765 : cluster 0 osdmap e65: 8 total, 8 up, 8 in 2024-03-07T03:08:49.716 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:49 smithi145 bash[21339]: audit 2024-03-07T03:08:48.388505+0000 mon.c (mon.2) 78 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-07T03:08:49.716 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:49 smithi145 bash[21339]: audit 2024-03-07T03:08:48.389895+0000 mon.a (mon.0) 766 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-07T03:08:49.716 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:49 smithi145 bash[21339]: debug 2024-03-07T03:08:49.398+0000 7f2333039700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-07T03:08:49.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:49 smithi189 bash[23614]: audit 2024-03-07T03:08:47.714133+0000 mgr.y (mgr.24302) 119 : audit 0 from='client.24494 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:49.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:49 smithi189 bash[23614]: audit 2024-03-07T03:08:48.377681+0000 mon.a (mon.0) 764 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-07T03:08:49.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:49 smithi189 bash[23614]: cluster 2024-03-07T03:08:48.377874+0000 mon.a (mon.0) 765 : cluster 0 osdmap e65: 8 total, 8 up, 8 in 2024-03-07T03:08:49.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:49 smithi189 bash[23614]: audit 2024-03-07T03:08:48.388505+0000 mon.c (mon.2) 78 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-07T03:08:49.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:49 smithi189 bash[23614]: audit 2024-03-07T03:08:48.389895+0000 mon.a (mon.0) 766 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-07T03:08:50.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: cluster 2024-03-07T03:08:48.403232+0000 mgr.y (mgr.24302) 120 : cluster 0 pgmap v91: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:50.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: cluster 2024-03-07T03:08:49.370012+0000 mon.a (mon.0) 767 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:08:50.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: audit 2024-03-07T03:08:49.390225+0000 mon.a (mon.0) 768 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-07T03:08:50.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: cluster 2024-03-07T03:08:49.390435+0000 mon.a (mon.0) 769 : cluster 0 osdmap e66: 8 total, 8 up, 8 in 2024-03-07T03:08:50.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: audit 2024-03-07T03:08:49.402625+0000 mon.c (mon.2) 79 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-07T03:08:50.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: audit 2024-03-07T03:08:49.403673+0000 mon.a (mon.0) 770 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-07T03:08:50.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: cluster 2024-03-07T03:08:49.405889+0000 mon.a (mon.0) 771 : cluster 4 Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-07T03:08:50.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: cluster 2024-03-07T03:08:49.406387+0000 mon.a (mon.0) 772 : cluster 3 Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-07T03:08:50.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: cluster 2024-03-07T03:08:49.426490+0000 mon.a (mon.0) 773 : cluster 0 osdmap e67: 8 total, 8 up, 8 in 2024-03-07T03:08:50.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: audit 2024-03-07T03:08:49.426738+0000 mon.a (mon.0) 774 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-07T03:08:50.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: cluster 2024-03-07T03:08:49.427427+0000 mon.a (mon.0) 775 : cluster 0 fsmap foo:0 2024-03-07T03:08:50.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: audit 2024-03-07T03:08:49.448623+0000 mon.a (mon.0) 776 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:50.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: audit 2024-03-07T03:08:49.492328+0000 mon.c (mon.2) 80 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:50.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: audit 2024-03-07T03:08:49.493976+0000 mon.c (mon.2) 81 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:50.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: audit 2024-03-07T03:08:49.495107+0000 mon.c (mon.2) 82 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:50.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: audit 2024-03-07T03:08:49.500581+0000 mon.a (mon.0) 777 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:50.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: audit 2024-03-07T03:08:49.507088+0000 mon.c (mon.2) 83 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi145.zmgvgw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:08:50.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: audit 2024-03-07T03:08:49.507523+0000 mon.a (mon.0) 778 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi145.zmgvgw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:08:50.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: audit 2024-03-07T03:08:49.511674+0000 mon.a (mon.0) 779 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi145.zmgvgw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-07T03:08:50.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[21339]: audit 2024-03-07T03:08:49.512658+0000 mon.c (mon.2) 84 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:50.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: cluster 2024-03-07T03:08:48.403232+0000 mgr.y (mgr.24302) 120 : cluster 0 pgmap v91: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:50.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: cluster 2024-03-07T03:08:49.370012+0000 mon.a (mon.0) 767 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:08:50.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: audit 2024-03-07T03:08:49.390225+0000 mon.a (mon.0) 768 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-07T03:08:50.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: cluster 2024-03-07T03:08:49.390435+0000 mon.a (mon.0) 769 : cluster 0 osdmap e66: 8 total, 8 up, 8 in 2024-03-07T03:08:50.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: audit 2024-03-07T03:08:49.402625+0000 mon.c (mon.2) 79 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-07T03:08:50.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: audit 2024-03-07T03:08:49.403673+0000 mon.a (mon.0) 770 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-07T03:08:50.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: cluster 2024-03-07T03:08:49.405889+0000 mon.a (mon.0) 771 : cluster 4 Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-07T03:08:50.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: cluster 2024-03-07T03:08:49.406387+0000 mon.a (mon.0) 772 : cluster 3 Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-07T03:08:50.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: cluster 2024-03-07T03:08:49.426490+0000 mon.a (mon.0) 773 : cluster 0 osdmap e67: 8 total, 8 up, 8 in 2024-03-07T03:08:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: audit 2024-03-07T03:08:49.426738+0000 mon.a (mon.0) 774 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-07T03:08:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: cluster 2024-03-07T03:08:49.427427+0000 mon.a (mon.0) 775 : cluster 0 fsmap foo:0 2024-03-07T03:08:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: audit 2024-03-07T03:08:49.448623+0000 mon.a (mon.0) 776 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: audit 2024-03-07T03:08:49.492328+0000 mon.c (mon.2) 80 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: audit 2024-03-07T03:08:49.493976+0000 mon.c (mon.2) 81 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: audit 2024-03-07T03:08:49.495107+0000 mon.c (mon.2) 82 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: audit 2024-03-07T03:08:49.500581+0000 mon.a (mon.0) 777 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: audit 2024-03-07T03:08:49.507088+0000 mon.c (mon.2) 83 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi145.zmgvgw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:08:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: audit 2024-03-07T03:08:49.507523+0000 mon.a (mon.0) 778 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi145.zmgvgw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:08:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: audit 2024-03-07T03:08:49.511674+0000 mon.a (mon.0) 779 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi145.zmgvgw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-07T03:08:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:50 smithi145 bash[30906]: audit 2024-03-07T03:08:49.512658+0000 mon.c (mon.2) 84 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: cluster 2024-03-07T03:08:48.403232+0000 mgr.y (mgr.24302) 120 : cluster 0 pgmap v91: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: cluster 2024-03-07T03:08:49.370012+0000 mon.a (mon.0) 767 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:08:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: audit 2024-03-07T03:08:49.390225+0000 mon.a (mon.0) 768 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-07T03:08:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: cluster 2024-03-07T03:08:49.390435+0000 mon.a (mon.0) 769 : cluster 0 osdmap e66: 8 total, 8 up, 8 in 2024-03-07T03:08:50.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: audit 2024-03-07T03:08:49.402625+0000 mon.c (mon.2) 79 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: audit 2024-03-07T03:08:49.403673+0000 mon.a (mon.0) 770 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: cluster 2024-03-07T03:08:49.405889+0000 mon.a (mon.0) 771 : cluster 4 Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: cluster 2024-03-07T03:08:49.406387+0000 mon.a (mon.0) 772 : cluster 3 Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: cluster 2024-03-07T03:08:49.426490+0000 mon.a (mon.0) 773 : cluster 0 osdmap e67: 8 total, 8 up, 8 in 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: audit 2024-03-07T03:08:49.426738+0000 mon.a (mon.0) 774 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: cluster 2024-03-07T03:08:49.427427+0000 mon.a (mon.0) 775 : cluster 0 fsmap foo:0 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: audit 2024-03-07T03:08:49.448623+0000 mon.a (mon.0) 776 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: audit 2024-03-07T03:08:49.492328+0000 mon.c (mon.2) 80 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: audit 2024-03-07T03:08:49.493976+0000 mon.c (mon.2) 81 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: audit 2024-03-07T03:08:49.495107+0000 mon.c (mon.2) 82 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: audit 2024-03-07T03:08:49.500581+0000 mon.a (mon.0) 777 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: audit 2024-03-07T03:08:49.507088+0000 mon.c (mon.2) 83 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi145.zmgvgw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: audit 2024-03-07T03:08:49.507523+0000 mon.a (mon.0) 778 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi145.zmgvgw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:08:50.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: audit 2024-03-07T03:08:49.511674+0000 mon.a (mon.0) 779 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi145.zmgvgw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-07T03:08:50.836 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:50 smithi189 bash[23614]: audit 2024-03-07T03:08:49.512658+0000 mon.c (mon.2) 84 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:51.036 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-03-07T03:08:51.706 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:51 smithi145 bash[30906]: cephadm 2024-03-07T03:08:49.433792+0000 mgr.y (mgr.24302) 121 : cephadm 1 Saving service mds.foo spec with placement count:2 2024-03-07T03:08:51.707 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:51 smithi145 bash[30906]: cephadm 2024-03-07T03:08:49.513645+0000 mgr.y (mgr.24302) 122 : cephadm 1 Deploying daemon mds.foo.smithi145.zmgvgw on smithi145 2024-03-07T03:08:51.707 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:51 smithi145 bash[30906]: cluster 2024-03-07T03:08:50.422437+0000 mon.a (mon.0) 780 : cluster 0 osdmap e68: 8 total, 8 up, 8 in 2024-03-07T03:08:51.707 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:51 smithi145 bash[21339]: cephadm 2024-03-07T03:08:49.433792+0000 mgr.y (mgr.24302) 121 : cephadm 1 Saving service mds.foo spec with placement count:2 2024-03-07T03:08:51.707 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:51 smithi145 bash[21339]: cephadm 2024-03-07T03:08:49.513645+0000 mgr.y (mgr.24302) 122 : cephadm 1 Deploying daemon mds.foo.smithi145.zmgvgw on smithi145 2024-03-07T03:08:51.707 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:51 smithi145 bash[21339]: cluster 2024-03-07T03:08:50.422437+0000 mon.a (mon.0) 780 : cluster 0 osdmap e68: 8 total, 8 up, 8 in 2024-03-07T03:08:51.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:51 smithi189 bash[23614]: cephadm 2024-03-07T03:08:49.433792+0000 mgr.y (mgr.24302) 121 : cephadm 1 Saving service mds.foo spec with placement count:2 2024-03-07T03:08:51.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:51 smithi189 bash[23614]: cephadm 2024-03-07T03:08:49.513645+0000 mgr.y (mgr.24302) 122 : cephadm 1 Deploying daemon mds.foo.smithi145.zmgvgw on smithi145 2024-03-07T03:08:51.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:51 smithi189 bash[23614]: cluster 2024-03-07T03:08:50.422437+0000 mon.a (mon.0) 780 : cluster 0 osdmap e68: 8 total, 8 up, 8 in 2024-03-07T03:08:52.731 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:08:52 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:08:52] "GET /metrics HTTP/1.1" 200 28558 "" "Prometheus/2.43.0" 2024-03-07T03:08:52.732 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:52 smithi145 bash[30906]: cluster 2024-03-07T03:08:50.403877+0000 mgr.y (mgr.24302) 123 : cluster 0 pgmap v94: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:52.732 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:52 smithi145 bash[30906]: cluster 2024-03-07T03:08:51.384011+0000 mon.a (mon.0) 781 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:08:52.732 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:52 smithi145 bash[30906]: cluster 2024-03-07T03:08:51.425832+0000 mon.a (mon.0) 782 : cluster 0 osdmap e69: 8 total, 8 up, 8 in 2024-03-07T03:08:52.732 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:52 smithi145 bash[21339]: cluster 2024-03-07T03:08:50.403877+0000 mgr.y (mgr.24302) 123 : cluster 0 pgmap v94: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:52.732 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:52 smithi145 bash[21339]: cluster 2024-03-07T03:08:51.384011+0000 mon.a (mon.0) 781 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:08:52.732 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:52 smithi145 bash[21339]: cluster 2024-03-07T03:08:51.425832+0000 mon.a (mon.0) 782 : cluster 0 osdmap e69: 8 total, 8 up, 8 in 2024-03-07T03:08:52.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:52 smithi189 bash[23614]: cluster 2024-03-07T03:08:50.403877+0000 mgr.y (mgr.24302) 123 : cluster 0 pgmap v94: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:52.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:52 smithi189 bash[23614]: cluster 2024-03-07T03:08:51.384011+0000 mon.a (mon.0) 781 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:08:52.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:52 smithi189 bash[23614]: cluster 2024-03-07T03:08:51.425832+0000 mon.a (mon.0) 782 : cluster 0 osdmap e69: 8 total, 8 up, 8 in 2024-03-07T03:08:53.017 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:08:52 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.018 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:08:52 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.018 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:52 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.018 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:52 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.018 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:08:52 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.019 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:08:52 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.019 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:08:52 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.019 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:08:52 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.020 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:08:52 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.311 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:08:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.312 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:08:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.312 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:08:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.312 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:08:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.313 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:08:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.313 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:08:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:53.313 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:08:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:54.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[21339]: cluster 2024-03-07T03:08:52.404474+0000 mgr.y (mgr.24302) 124 : cluster 0 pgmap v97: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:54.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[21339]: audit 2024-03-07T03:08:53.307459+0000 mon.a (mon.0) 783 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:54.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[21339]: audit 2024-03-07T03:08:53.314618+0000 mon.a (mon.0) 784 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:54.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[21339]: audit 2024-03-07T03:08:53.339855+0000 mon.a (mon.0) 785 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:54.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[21339]: audit 2024-03-07T03:08:53.340619+0000 mon.c (mon.2) 85 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.zkinbe", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:08:54.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[21339]: audit 2024-03-07T03:08:53.340928+0000 mon.a (mon.0) 786 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.zkinbe", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:08:54.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[21339]: audit 2024-03-07T03:08:53.344501+0000 mon.a (mon.0) 787 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.zkinbe", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-07T03:08:54.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[21339]: audit 2024-03-07T03:08:53.345381+0000 mon.c (mon.2) 86 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:54.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[21339]: cephadm 2024-03-07T03:08:53.346349+0000 mgr.y (mgr.24302) 125 : cephadm 1 Deploying daemon mds.foo.smithi189.zkinbe on smithi189 2024-03-07T03:08:54.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[30906]: cluster 2024-03-07T03:08:52.404474+0000 mgr.y (mgr.24302) 124 : cluster 0 pgmap v97: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:54.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[30906]: audit 2024-03-07T03:08:53.307459+0000 mon.a (mon.0) 783 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:54.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[30906]: audit 2024-03-07T03:08:53.314618+0000 mon.a (mon.0) 784 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:54.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[30906]: audit 2024-03-07T03:08:53.339855+0000 mon.a (mon.0) 785 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:54.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[30906]: audit 2024-03-07T03:08:53.340619+0000 mon.c (mon.2) 85 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.zkinbe", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:08:54.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[30906]: audit 2024-03-07T03:08:53.340928+0000 mon.a (mon.0) 786 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.zkinbe", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:08:54.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[30906]: audit 2024-03-07T03:08:53.344501+0000 mon.a (mon.0) 787 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.zkinbe", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-07T03:08:54.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[30906]: audit 2024-03-07T03:08:53.345381+0000 mon.c (mon.2) 86 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:54.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:54 smithi145 bash[30906]: cephadm 2024-03-07T03:08:53.346349+0000 mgr.y (mgr.24302) 125 : cephadm 1 Deploying daemon mds.foo.smithi189.zkinbe on smithi189 2024-03-07T03:08:54.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:54 smithi189 bash[23614]: cluster 2024-03-07T03:08:52.404474+0000 mgr.y (mgr.24302) 124 : cluster 0 pgmap v97: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:54.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:54 smithi189 bash[23614]: audit 2024-03-07T03:08:53.307459+0000 mon.a (mon.0) 783 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:54.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:54 smithi189 bash[23614]: audit 2024-03-07T03:08:53.314618+0000 mon.a (mon.0) 784 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:54.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:54 smithi189 bash[23614]: audit 2024-03-07T03:08:53.339855+0000 mon.a (mon.0) 785 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:54.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:54 smithi189 bash[23614]: audit 2024-03-07T03:08:53.340619+0000 mon.c (mon.2) 85 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.zkinbe", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:08:54.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:54 smithi189 bash[23614]: audit 2024-03-07T03:08:53.340928+0000 mon.a (mon.0) 786 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.zkinbe", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:08:54.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:54 smithi189 bash[23614]: audit 2024-03-07T03:08:53.344501+0000 mon.a (mon.0) 787 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.zkinbe", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-07T03:08:54.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:54 smithi189 bash[23614]: audit 2024-03-07T03:08:53.345381+0000 mon.c (mon.2) 86 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:54.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:54 smithi189 bash[23614]: cephadm 2024-03-07T03:08:53.346349+0000 mgr.y (mgr.24302) 125 : cephadm 1 Deploying daemon mds.foo.smithi189.zkinbe on smithi189 2024-03-07T03:08:55.391 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.391 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.391 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.392 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.392 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.392 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.393 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.393 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.393 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.683 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.683 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.683 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.683 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.684 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.684 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.684 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.684 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:55.685 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:08:55 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:56.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: cluster 2024-03-07T03:08:54.405492+0000 mgr.y (mgr.24302) 126 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:56.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: audit 2024-03-07T03:08:55.312135+0000 mon.c (mon.2) 87 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:08:56.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: cluster 2024-03-07T03:08:55.313125+0000 mon.a (mon.0) 788 : cluster 0 mds.? [v2:172.21.15.145:6834/3433351665,v1:172.21.15.145:6835/3433351665] up:boot 2024-03-07T03:08:56.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: cluster 2024-03-07T03:08:55.313291+0000 mon.a (mon.0) 789 : cluster 1 daemon mds.foo.smithi145.zmgvgw assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-07T03:08:56.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: cluster 2024-03-07T03:08:55.313482+0000 mon.a (mon.0) 790 : cluster 1 Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-07T03:08:56.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: cluster 2024-03-07T03:08:55.313556+0000 mon.a (mon.0) 791 : cluster 1 Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-07T03:08:56.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: cluster 2024-03-07T03:08:55.313596+0000 mon.a (mon.0) 792 : cluster 1 Cluster is now healthy 2024-03-07T03:08:56.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: cluster 2024-03-07T03:08:55.314128+0000 mon.a (mon.0) 793 : cluster 0 fsmap foo:0 1 up:standby 2024-03-07T03:08:56.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: cluster 2024-03-07T03:08:55.318866+0000 mon.a (mon.0) 794 : cluster 0 fsmap foo:1 {0=foo.smithi145.zmgvgw=up:creating} 2024-03-07T03:08:56.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: cluster 2024-03-07T03:08:55.347821+0000 mon.a (mon.0) 795 : cluster 1 daemon mds.foo.smithi145.zmgvgw is now active in filesystem foo as rank 0 2024-03-07T03:08:56.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: audit 2024-03-07T03:08:55.691777+0000 mon.a (mon.0) 796 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: audit 2024-03-07T03:08:55.709285+0000 mon.a (mon.0) 797 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: audit 2024-03-07T03:08:55.722689+0000 mon.a (mon.0) 798 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: audit 2024-03-07T03:08:55.735634+0000 mon.a (mon.0) 799 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: audit 2024-03-07T03:08:55.746551+0000 mon.a (mon.0) 800 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[21339]: audit 2024-03-07T03:08:55.816642+0000 mon.c (mon.2) 88 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: cluster 2024-03-07T03:08:54.405492+0000 mgr.y (mgr.24302) 126 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: audit 2024-03-07T03:08:55.312135+0000 mon.c (mon.2) 87 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: cluster 2024-03-07T03:08:55.313125+0000 mon.a (mon.0) 788 : cluster 0 mds.? [v2:172.21.15.145:6834/3433351665,v1:172.21.15.145:6835/3433351665] up:boot 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: cluster 2024-03-07T03:08:55.313291+0000 mon.a (mon.0) 789 : cluster 1 daemon mds.foo.smithi145.zmgvgw assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: cluster 2024-03-07T03:08:55.313482+0000 mon.a (mon.0) 790 : cluster 1 Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: cluster 2024-03-07T03:08:55.313556+0000 mon.a (mon.0) 791 : cluster 1 Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: cluster 2024-03-07T03:08:55.313596+0000 mon.a (mon.0) 792 : cluster 1 Cluster is now healthy 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: cluster 2024-03-07T03:08:55.314128+0000 mon.a (mon.0) 793 : cluster 0 fsmap foo:0 1 up:standby 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: cluster 2024-03-07T03:08:55.318866+0000 mon.a (mon.0) 794 : cluster 0 fsmap foo:1 {0=foo.smithi145.zmgvgw=up:creating} 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: cluster 2024-03-07T03:08:55.347821+0000 mon.a (mon.0) 795 : cluster 1 daemon mds.foo.smithi145.zmgvgw is now active in filesystem foo as rank 0 2024-03-07T03:08:56.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: audit 2024-03-07T03:08:55.691777+0000 mon.a (mon.0) 796 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: audit 2024-03-07T03:08:55.709285+0000 mon.a (mon.0) 797 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: audit 2024-03-07T03:08:55.722689+0000 mon.a (mon.0) 798 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: audit 2024-03-07T03:08:55.735634+0000 mon.a (mon.0) 799 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: audit 2024-03-07T03:08:55.746551+0000 mon.a (mon.0) 800 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:56 smithi145 bash[30906]: audit 2024-03-07T03:08:55.816642+0000 mon.c (mon.2) 88 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:56.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: cluster 2024-03-07T03:08:54.405492+0000 mgr.y (mgr.24302) 126 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:56.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: audit 2024-03-07T03:08:55.312135+0000 mon.c (mon.2) 87 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:08:56.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: cluster 2024-03-07T03:08:55.313125+0000 mon.a (mon.0) 788 : cluster 0 mds.? [v2:172.21.15.145:6834/3433351665,v1:172.21.15.145:6835/3433351665] up:boot 2024-03-07T03:08:56.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: cluster 2024-03-07T03:08:55.313291+0000 mon.a (mon.0) 789 : cluster 1 daemon mds.foo.smithi145.zmgvgw assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-07T03:08:56.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: cluster 2024-03-07T03:08:55.313482+0000 mon.a (mon.0) 790 : cluster 1 Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-07T03:08:56.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: cluster 2024-03-07T03:08:55.313556+0000 mon.a (mon.0) 791 : cluster 1 Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-07T03:08:56.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: cluster 2024-03-07T03:08:55.313596+0000 mon.a (mon.0) 792 : cluster 1 Cluster is now healthy 2024-03-07T03:08:56.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: cluster 2024-03-07T03:08:55.314128+0000 mon.a (mon.0) 793 : cluster 0 fsmap foo:0 1 up:standby 2024-03-07T03:08:56.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: cluster 2024-03-07T03:08:55.318866+0000 mon.a (mon.0) 794 : cluster 0 fsmap foo:1 {0=foo.smithi145.zmgvgw=up:creating} 2024-03-07T03:08:56.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: cluster 2024-03-07T03:08:55.347821+0000 mon.a (mon.0) 795 : cluster 1 daemon mds.foo.smithi145.zmgvgw is now active in filesystem foo as rank 0 2024-03-07T03:08:56.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: audit 2024-03-07T03:08:55.691777+0000 mon.a (mon.0) 796 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: audit 2024-03-07T03:08:55.709285+0000 mon.a (mon.0) 797 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: audit 2024-03-07T03:08:55.722689+0000 mon.a (mon.0) 798 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: audit 2024-03-07T03:08:55.735634+0000 mon.a (mon.0) 799 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: audit 2024-03-07T03:08:55.746551+0000 mon.a (mon.0) 800 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:56.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:56 smithi189 bash[23614]: audit 2024-03-07T03:08:55.816642+0000 mon.c (mon.2) 88 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:56.790 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:08:57.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:57 smithi189 bash[23614]: cluster 2024-03-07T03:08:56.327063+0000 mon.a (mon.0) 801 : cluster 0 mds.? [v2:172.21.15.145:6834/3433351665,v1:172.21.15.145:6835/3433351665] up:active 2024-03-07T03:08:57.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:57 smithi189 bash[23614]: cluster 2024-03-07T03:08:56.327190+0000 mon.a (mon.0) 802 : cluster 0 fsmap foo:1 {0=foo.smithi145.zmgvgw=up:active} 2024-03-07T03:08:57.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:57 smithi145 bash[21339]: cluster 2024-03-07T03:08:56.327063+0000 mon.a (mon.0) 801 : cluster 0 mds.? [v2:172.21.15.145:6834/3433351665,v1:172.21.15.145:6835/3433351665] up:active 2024-03-07T03:08:57.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:57 smithi145 bash[21339]: cluster 2024-03-07T03:08:56.327190+0000 mon.a (mon.0) 802 : cluster 0 fsmap foo:1 {0=foo.smithi145.zmgvgw=up:active} 2024-03-07T03:08:57.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:57 smithi145 bash[30906]: cluster 2024-03-07T03:08:56.327063+0000 mon.a (mon.0) 801 : cluster 0 mds.? [v2:172.21.15.145:6834/3433351665,v1:172.21.15.145:6835/3433351665] up:active 2024-03-07T03:08:57.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:57 smithi145 bash[30906]: cluster 2024-03-07T03:08:56.327190+0000 mon.a (mon.0) 802 : cluster 0 fsmap foo:1 {0=foo.smithi145.zmgvgw=up:active} 2024-03-07T03:08:58.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:58 smithi189 bash[23614]: cluster 2024-03-07T03:08:56.406102+0000 mgr.y (mgr.24302) 127 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:58.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:58 smithi189 bash[23614]: cluster 2024-03-07T03:08:57.328098+0000 mon.a (mon.0) 803 : cluster 0 mds.? [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] up:boot 2024-03-07T03:08:58.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:58 smithi189 bash[23614]: cluster 2024-03-07T03:08:57.328197+0000 mon.a (mon.0) 804 : cluster 1 Dropping low affinity active daemon mds.foo.smithi145.zmgvgw in favor of higher affinity standby. 2024-03-07T03:08:58.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:58 smithi189 bash[23614]: cluster 2024-03-07T03:08:57.328253+0000 mon.a (mon.0) 805 : cluster 3 Replacing daemon mds.foo.smithi145.zmgvgw as rank 0 with standby daemon mds.foo.smithi189.zkinbe 2024-03-07T03:08:58.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:58 smithi189 bash[23614]: cluster 2024-03-07T03:08:57.328951+0000 mon.a (mon.0) 806 : cluster 3 Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-07T03:08:58.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:58 smithi189 bash[23614]: cluster 2024-03-07T03:08:57.328993+0000 mon.a (mon.0) 807 : cluster 3 Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-07T03:08:58.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:58 smithi189 bash[23614]: cluster 2024-03-07T03:08:57.329023+0000 mon.a (mon.0) 808 : cluster 0 fsmap foo:1 {0=foo.smithi145.zmgvgw=up:active} 1 up:standby 2024-03-07T03:08:58.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:58 smithi189 bash[23614]: audit 2024-03-07T03:08:57.331194+0000 mon.c (mon.2) 89 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.zkinbe"}]: dispatch 2024-03-07T03:08:58.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:58 smithi189 bash[23614]: cluster 2024-03-07T03:08:57.341328+0000 mon.a (mon.0) 809 : cluster 0 osdmap e70: 8 total, 8 up, 8 in 2024-03-07T03:08:58.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:58 smithi189 bash[23614]: cluster 2024-03-07T03:08:57.345651+0000 mon.a (mon.0) 810 : cluster 0 fsmap foo:1/1 {0=foo.smithi189.zkinbe=up:replay} 2024-03-07T03:08:58.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:58 smithi189 bash[23614]: audit 2024-03-07T03:08:57.825193+0000 mon.a (mon.0) 811 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:58.651 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[30906]: cluster 2024-03-07T03:08:56.406102+0000 mgr.y (mgr.24302) 127 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:58.651 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[30906]: cluster 2024-03-07T03:08:57.328098+0000 mon.a (mon.0) 803 : cluster 0 mds.? [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] up:boot 2024-03-07T03:08:58.651 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[30906]: cluster 2024-03-07T03:08:57.328197+0000 mon.a (mon.0) 804 : cluster 1 Dropping low affinity active daemon mds.foo.smithi145.zmgvgw in favor of higher affinity standby. 2024-03-07T03:08:58.651 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[30906]: cluster 2024-03-07T03:08:57.328253+0000 mon.a (mon.0) 805 : cluster 3 Replacing daemon mds.foo.smithi145.zmgvgw as rank 0 with standby daemon mds.foo.smithi189.zkinbe 2024-03-07T03:08:58.651 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[30906]: cluster 2024-03-07T03:08:57.328951+0000 mon.a (mon.0) 806 : cluster 3 Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-07T03:08:58.651 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[30906]: cluster 2024-03-07T03:08:57.328993+0000 mon.a (mon.0) 807 : cluster 3 Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-07T03:08:58.651 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[30906]: cluster 2024-03-07T03:08:57.329023+0000 mon.a (mon.0) 808 : cluster 0 fsmap foo:1 {0=foo.smithi145.zmgvgw=up:active} 1 up:standby 2024-03-07T03:08:58.651 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[30906]: audit 2024-03-07T03:08:57.331194+0000 mon.c (mon.2) 89 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.zkinbe"}]: dispatch 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[30906]: cluster 2024-03-07T03:08:57.341328+0000 mon.a (mon.0) 809 : cluster 0 osdmap e70: 8 total, 8 up, 8 in 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[30906]: cluster 2024-03-07T03:08:57.345651+0000 mon.a (mon.0) 810 : cluster 0 fsmap foo:1/1 {0=foo.smithi189.zkinbe=up:replay} 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[30906]: audit 2024-03-07T03:08:57.825193+0000 mon.a (mon.0) 811 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[21339]: cluster 2024-03-07T03:08:56.406102+0000 mgr.y (mgr.24302) 127 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[21339]: cluster 2024-03-07T03:08:57.328098+0000 mon.a (mon.0) 803 : cluster 0 mds.? [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] up:boot 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[21339]: cluster 2024-03-07T03:08:57.328197+0000 mon.a (mon.0) 804 : cluster 1 Dropping low affinity active daemon mds.foo.smithi145.zmgvgw in favor of higher affinity standby. 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[21339]: cluster 2024-03-07T03:08:57.328253+0000 mon.a (mon.0) 805 : cluster 3 Replacing daemon mds.foo.smithi145.zmgvgw as rank 0 with standby daemon mds.foo.smithi189.zkinbe 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[21339]: cluster 2024-03-07T03:08:57.328951+0000 mon.a (mon.0) 806 : cluster 3 Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[21339]: cluster 2024-03-07T03:08:57.328993+0000 mon.a (mon.0) 807 : cluster 3 Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[21339]: cluster 2024-03-07T03:08:57.329023+0000 mon.a (mon.0) 808 : cluster 0 fsmap foo:1 {0=foo.smithi145.zmgvgw=up:active} 1 up:standby 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[21339]: audit 2024-03-07T03:08:57.331194+0000 mon.c (mon.2) 89 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.zkinbe"}]: dispatch 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[21339]: cluster 2024-03-07T03:08:57.341328+0000 mon.a (mon.0) 809 : cluster 0 osdmap e70: 8 total, 8 up, 8 in 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[21339]: cluster 2024-03-07T03:08:57.345651+0000 mon.a (mon.0) 810 : cluster 0 fsmap foo:1/1 {0=foo.smithi189.zkinbe=up:replay} 2024-03-07T03:08:58.652 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:58 smithi145 bash[21339]: audit 2024-03-07T03:08:57.825193+0000 mon.a (mon.0) 811 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:08:59.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:59 smithi189 bash[23614]: cluster 2024-03-07T03:08:58.342946+0000 mon.a (mon.0) 812 : cluster 1 Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-07T03:08:59.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:59 smithi189 bash[23614]: cluster 2024-03-07T03:08:58.349625+0000 mon.a (mon.0) 813 : cluster 0 mds.? [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] up:reconnect 2024-03-07T03:08:59.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:59 smithi189 bash[23614]: cluster 2024-03-07T03:08:58.349827+0000 mon.a (mon.0) 814 : cluster 0 mds.? [v2:172.21.15.145:6834/4165785656,v1:172.21.15.145:6835/4165785656] up:boot 2024-03-07T03:08:59.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:59 smithi189 bash[23614]: cluster 2024-03-07T03:08:58.349935+0000 mon.a (mon.0) 815 : cluster 0 fsmap foo:1/1 {0=foo.smithi189.zkinbe=up:reconnect} 1 up:standby 2024-03-07T03:08:59.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:08:59 smithi189 bash[23614]: audit 2024-03-07T03:08:58.351122+0000 mon.c (mon.2) 90 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:08:59.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:59 smithi145 bash[21339]: cluster 2024-03-07T03:08:58.342946+0000 mon.a (mon.0) 812 : cluster 1 Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-07T03:08:59.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:59 smithi145 bash[21339]: cluster 2024-03-07T03:08:58.349625+0000 mon.a (mon.0) 813 : cluster 0 mds.? [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] up:reconnect 2024-03-07T03:08:59.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:59 smithi145 bash[21339]: cluster 2024-03-07T03:08:58.349827+0000 mon.a (mon.0) 814 : cluster 0 mds.? [v2:172.21.15.145:6834/4165785656,v1:172.21.15.145:6835/4165785656] up:boot 2024-03-07T03:08:59.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:59 smithi145 bash[21339]: cluster 2024-03-07T03:08:58.349935+0000 mon.a (mon.0) 815 : cluster 0 fsmap foo:1/1 {0=foo.smithi189.zkinbe=up:reconnect} 1 up:standby 2024-03-07T03:08:59.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:08:59 smithi145 bash[21339]: audit 2024-03-07T03:08:58.351122+0000 mon.c (mon.2) 90 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:08:59.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:59 smithi145 bash[30906]: cluster 2024-03-07T03:08:58.342946+0000 mon.a (mon.0) 812 : cluster 1 Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-07T03:08:59.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:59 smithi145 bash[30906]: cluster 2024-03-07T03:08:58.349625+0000 mon.a (mon.0) 813 : cluster 0 mds.? [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] up:reconnect 2024-03-07T03:08:59.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:59 smithi145 bash[30906]: cluster 2024-03-07T03:08:58.349827+0000 mon.a (mon.0) 814 : cluster 0 mds.? [v2:172.21.15.145:6834/4165785656,v1:172.21.15.145:6835/4165785656] up:boot 2024-03-07T03:08:59.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:59 smithi145 bash[30906]: cluster 2024-03-07T03:08:58.349935+0000 mon.a (mon.0) 815 : cluster 0 fsmap foo:1/1 {0=foo.smithi189.zkinbe=up:reconnect} 1 up:standby 2024-03-07T03:08:59.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:08:59 smithi145 bash[30906]: audit 2024-03-07T03:08:58.351122+0000 mon.c (mon.2) 90 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:09:00.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:00 smithi145 bash[21339]: cluster 2024-03-07T03:08:58.406672+0000 mgr.y (mgr.24302) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:00.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:00 smithi145 bash[21339]: cluster 2024-03-07T03:08:59.352163+0000 mon.a (mon.0) 816 : cluster 0 mds.? [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] up:rejoin 2024-03-07T03:09:00.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:00 smithi145 bash[21339]: cluster 2024-03-07T03:08:59.352356+0000 mon.a (mon.0) 817 : cluster 0 fsmap foo:1/1 {0=foo.smithi189.zkinbe=up:rejoin} 1 up:standby 2024-03-07T03:09:00.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:00 smithi145 bash[21339]: cluster 2024-03-07T03:08:59.352868+0000 mon.a (mon.0) 818 : cluster 1 daemon mds.foo.smithi189.zkinbe is now active in filesystem foo as rank 0 2024-03-07T03:09:00.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:00 smithi145 bash[30906]: cluster 2024-03-07T03:08:58.406672+0000 mgr.y (mgr.24302) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:00.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:00 smithi145 bash[30906]: cluster 2024-03-07T03:08:59.352163+0000 mon.a (mon.0) 816 : cluster 0 mds.? [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] up:rejoin 2024-03-07T03:09:00.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:00 smithi145 bash[30906]: cluster 2024-03-07T03:08:59.352356+0000 mon.a (mon.0) 817 : cluster 0 fsmap foo:1/1 {0=foo.smithi189.zkinbe=up:rejoin} 1 up:standby 2024-03-07T03:09:00.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:00 smithi145 bash[30906]: cluster 2024-03-07T03:08:59.352868+0000 mon.a (mon.0) 818 : cluster 1 daemon mds.foo.smithi189.zkinbe is now active in filesystem foo as rank 0 2024-03-07T03:09:00.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:00 smithi189 bash[23614]: cluster 2024-03-07T03:08:58.406672+0000 mgr.y (mgr.24302) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:00.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:00 smithi189 bash[23614]: cluster 2024-03-07T03:08:59.352163+0000 mon.a (mon.0) 816 : cluster 0 mds.? [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] up:rejoin 2024-03-07T03:09:00.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:00 smithi189 bash[23614]: cluster 2024-03-07T03:08:59.352356+0000 mon.a (mon.0) 817 : cluster 0 fsmap foo:1/1 {0=foo.smithi189.zkinbe=up:rejoin} 1 up:standby 2024-03-07T03:09:00.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:00 smithi189 bash[23614]: cluster 2024-03-07T03:08:59.352868+0000 mon.a (mon.0) 818 : cluster 1 daemon mds.foo.smithi189.zkinbe is now active in filesystem foo as rank 0 2024-03-07T03:09:01.300 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-03-07T03:09:01.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:01 smithi145 bash[21339]: cluster 2024-03-07T03:09:00.347269+0000 mon.a (mon.0) 819 : cluster 1 Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-07T03:09:01.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:01 smithi145 bash[21339]: cluster 2024-03-07T03:09:00.347299+0000 mon.a (mon.0) 820 : cluster 1 Cluster is now healthy 2024-03-07T03:09:01.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:01 smithi145 bash[21339]: cluster 2024-03-07T03:09:00.356350+0000 mon.a (mon.0) 821 : cluster 0 mds.? [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] up:active 2024-03-07T03:09:01.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:01 smithi145 bash[21339]: cluster 2024-03-07T03:09:00.356385+0000 mon.a (mon.0) 822 : cluster 0 fsmap foo:1 {0=foo.smithi189.zkinbe=up:active} 1 up:standby 2024-03-07T03:09:01.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:01 smithi145 bash[21339]: audit 2024-03-07T03:09:00.452546+0000 mon.a (mon.0) 823 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:01.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:01 smithi145 bash[21339]: audit 2024-03-07T03:09:00.460373+0000 mon.a (mon.0) 824 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:01.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:01 smithi145 bash[30906]: cluster 2024-03-07T03:09:00.347269+0000 mon.a (mon.0) 819 : cluster 1 Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-07T03:09:01.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:01 smithi145 bash[30906]: cluster 2024-03-07T03:09:00.347299+0000 mon.a (mon.0) 820 : cluster 1 Cluster is now healthy 2024-03-07T03:09:01.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:01 smithi145 bash[30906]: cluster 2024-03-07T03:09:00.356350+0000 mon.a (mon.0) 821 : cluster 0 mds.? [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] up:active 2024-03-07T03:09:01.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:01 smithi145 bash[30906]: cluster 2024-03-07T03:09:00.356385+0000 mon.a (mon.0) 822 : cluster 0 fsmap foo:1 {0=foo.smithi189.zkinbe=up:active} 1 up:standby 2024-03-07T03:09:01.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:01 smithi145 bash[30906]: audit 2024-03-07T03:09:00.452546+0000 mon.a (mon.0) 823 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:01.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:01 smithi145 bash[30906]: audit 2024-03-07T03:09:00.460373+0000 mon.a (mon.0) 824 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:01.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:01 smithi189 bash[23614]: cluster 2024-03-07T03:09:00.347269+0000 mon.a (mon.0) 819 : cluster 1 Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-07T03:09:01.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:01 smithi189 bash[23614]: cluster 2024-03-07T03:09:00.347299+0000 mon.a (mon.0) 820 : cluster 1 Cluster is now healthy 2024-03-07T03:09:01.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:01 smithi189 bash[23614]: cluster 2024-03-07T03:09:00.356350+0000 mon.a (mon.0) 821 : cluster 0 mds.? [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] up:active 2024-03-07T03:09:01.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:01 smithi189 bash[23614]: cluster 2024-03-07T03:09:00.356385+0000 mon.a (mon.0) 822 : cluster 0 fsmap foo:1 {0=foo.smithi189.zkinbe=up:active} 1 up:standby 2024-03-07T03:09:01.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:01 smithi189 bash[23614]: audit 2024-03-07T03:09:00.452546+0000 mon.a (mon.0) 823 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:01.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:01 smithi189 bash[23614]: audit 2024-03-07T03:09:00.460373+0000 mon.a (mon.0) 824 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:02.814 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:09:02] "GET /metrics HTTP/1.1" 200 28558 "" "Prometheus/2.43.0" 2024-03-07T03:09:02.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[21339]: cluster 2024-03-07T03:09:00.407233+0000 mgr.y (mgr.24302) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.6 KiB/s wr, 6 op/s 2024-03-07T03:09:02.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[21339]: audit 2024-03-07T03:09:01.614641+0000 mon.a (mon.0) 825 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:02.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[21339]: audit 2024-03-07T03:09:01.630589+0000 mon.a (mon.0) 826 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:02.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[21339]: audit 2024-03-07T03:09:01.634767+0000 mon.c (mon.2) 91 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:09:02.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[21339]: audit 2024-03-07T03:09:01.636800+0000 mon.c (mon.2) 92 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:09:02.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[21339]: audit 2024-03-07T03:09:01.647738+0000 mon.a (mon.0) 827 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:02.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[21339]: audit 2024-03-07T03:09:01.741014+0000 mon.c (mon.2) 93 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:09:02.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[21339]: audit 2024-03-07T03:09:01.744426+0000 mon.c (mon.2) 94 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:09:02.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[21339]: audit 2024-03-07T03:09:01.745979+0000 mon.c (mon.2) 95 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:09:02.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[21339]: audit 2024-03-07T03:09:01.754767+0000 mon.a (mon.0) 828 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:02.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[30906]: cluster 2024-03-07T03:09:00.407233+0000 mgr.y (mgr.24302) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.6 KiB/s wr, 6 op/s 2024-03-07T03:09:02.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[30906]: audit 2024-03-07T03:09:01.614641+0000 mon.a (mon.0) 825 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:02.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[30906]: audit 2024-03-07T03:09:01.630589+0000 mon.a (mon.0) 826 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:02.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[30906]: audit 2024-03-07T03:09:01.634767+0000 mon.c (mon.2) 91 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:09:02.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[30906]: audit 2024-03-07T03:09:01.636800+0000 mon.c (mon.2) 92 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:09:02.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[30906]: audit 2024-03-07T03:09:01.647738+0000 mon.a (mon.0) 827 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:02.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[30906]: audit 2024-03-07T03:09:01.741014+0000 mon.c (mon.2) 93 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:09:02.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[30906]: audit 2024-03-07T03:09:01.744426+0000 mon.c (mon.2) 94 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:09:02.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[30906]: audit 2024-03-07T03:09:01.745979+0000 mon.c (mon.2) 95 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:09:02.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:02 smithi145 bash[30906]: audit 2024-03-07T03:09:01.754767+0000 mon.a (mon.0) 828 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:02.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:02 smithi189 bash[23614]: cluster 2024-03-07T03:09:00.407233+0000 mgr.y (mgr.24302) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.6 KiB/s wr, 6 op/s 2024-03-07T03:09:02.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:02 smithi189 bash[23614]: audit 2024-03-07T03:09:01.614641+0000 mon.a (mon.0) 825 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:02.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:02 smithi189 bash[23614]: audit 2024-03-07T03:09:01.630589+0000 mon.a (mon.0) 826 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:02.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:02 smithi189 bash[23614]: audit 2024-03-07T03:09:01.634767+0000 mon.c (mon.2) 91 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:09:02.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:02 smithi189 bash[23614]: audit 2024-03-07T03:09:01.636800+0000 mon.c (mon.2) 92 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:09:02.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:02 smithi189 bash[23614]: audit 2024-03-07T03:09:01.647738+0000 mon.a (mon.0) 827 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:02.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:02 smithi189 bash[23614]: audit 2024-03-07T03:09:01.741014+0000 mon.c (mon.2) 93 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:09:02.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:02 smithi189 bash[23614]: audit 2024-03-07T03:09:01.744426+0000 mon.c (mon.2) 94 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:09:02.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:02 smithi189 bash[23614]: audit 2024-03-07T03:09:01.745979+0000 mon.c (mon.2) 95 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:09:02.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:02 smithi189 bash[23614]: audit 2024-03-07T03:09:01.754767+0000 mon.a (mon.0) 828 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:03.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:03 smithi189 bash[23614]: audit 2024-03-07T03:09:02.407508+0000 mon.c (mon.2) 96 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-07T03:09:03.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:03 smithi189 bash[23614]: audit 2024-03-07T03:09:02.408317+0000 mon.c (mon.2) 97 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-03-07T03:09:03.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:03 smithi189 bash[23614]: audit 2024-03-07T03:09:02.408445+0000 mon.a (mon.0) 829 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-07T03:09:03.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:03 smithi189 bash[23614]: audit 2024-03-07T03:09:02.409099+0000 mon.a (mon.0) 830 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-03-07T03:09:03.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:03 smithi189 bash[23614]: audit 2024-03-07T03:09:02.409145+0000 mon.c (mon.2) 98 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-07T03:09:03.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:03 smithi189 bash[23614]: audit 2024-03-07T03:09:02.410109+0000 mon.a (mon.0) 831 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-07T03:09:03.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:03 smithi189 bash[23614]: audit 2024-03-07T03:09:02.449858+0000 mon.c (mon.2) 99 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:03.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:03 smithi189 bash[23614]: audit 2024-03-07T03:09:02.839787+0000 mon.a (mon.0) 832 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:03.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[21339]: audit 2024-03-07T03:09:02.407508+0000 mon.c (mon.2) 96 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-07T03:09:03.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[21339]: audit 2024-03-07T03:09:02.408317+0000 mon.c (mon.2) 97 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-03-07T03:09:03.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[21339]: audit 2024-03-07T03:09:02.408445+0000 mon.a (mon.0) 829 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-07T03:09:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[21339]: audit 2024-03-07T03:09:02.409099+0000 mon.a (mon.0) 830 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-03-07T03:09:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[21339]: audit 2024-03-07T03:09:02.409145+0000 mon.c (mon.2) 98 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-07T03:09:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[21339]: audit 2024-03-07T03:09:02.410109+0000 mon.a (mon.0) 831 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-07T03:09:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[21339]: audit 2024-03-07T03:09:02.449858+0000 mon.c (mon.2) 99 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[21339]: audit 2024-03-07T03:09:02.839787+0000 mon.a (mon.0) 832 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:03.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[30906]: audit 2024-03-07T03:09:02.407508+0000 mon.c (mon.2) 96 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-07T03:09:03.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[30906]: audit 2024-03-07T03:09:02.408317+0000 mon.c (mon.2) 97 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-03-07T03:09:03.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[30906]: audit 2024-03-07T03:09:02.408445+0000 mon.a (mon.0) 829 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-07T03:09:03.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[30906]: audit 2024-03-07T03:09:02.409099+0000 mon.a (mon.0) 830 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-03-07T03:09:03.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[30906]: audit 2024-03-07T03:09:02.409145+0000 mon.c (mon.2) 98 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-07T03:09:03.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[30906]: audit 2024-03-07T03:09:02.410109+0000 mon.a (mon.0) 831 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-07T03:09:03.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[30906]: audit 2024-03-07T03:09:02.449858+0000 mon.c (mon.2) 99 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:03.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:03 smithi145 bash[30906]: audit 2024-03-07T03:09:02.839787+0000 mon.a (mon.0) 832 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:09:04.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:04 smithi145 bash[21339]: cluster 2024-03-07T03:09:02.407774+0000 mgr.y (mgr.24302) 130 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.4 KiB/s wr, 5 op/s 2024-03-07T03:09:04.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:04 smithi145 bash[21339]: audit 2024-03-07T03:09:03.379842+0000 mon.a (mon.0) 833 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-03-07T03:09:04.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:04 smithi145 bash[21339]: audit 2024-03-07T03:09:03.380100+0000 mon.a (mon.0) 834 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-03-07T03:09:04.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:04 smithi145 bash[21339]: audit 2024-03-07T03:09:03.380278+0000 mon.a (mon.0) 835 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-03-07T03:09:04.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:04 smithi145 bash[21339]: cluster 2024-03-07T03:09:03.380475+0000 mon.a (mon.0) 836 : cluster 0 osdmap e71: 8 total, 8 up, 8 in 2024-03-07T03:09:04.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:04 smithi145 bash[30906]: cluster 2024-03-07T03:09:02.407774+0000 mgr.y (mgr.24302) 130 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.4 KiB/s wr, 5 op/s 2024-03-07T03:09:04.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:04 smithi145 bash[30906]: audit 2024-03-07T03:09:03.379842+0000 mon.a (mon.0) 833 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-03-07T03:09:04.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:04 smithi145 bash[30906]: audit 2024-03-07T03:09:03.380100+0000 mon.a (mon.0) 834 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-03-07T03:09:04.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:04 smithi145 bash[30906]: audit 2024-03-07T03:09:03.380278+0000 mon.a (mon.0) 835 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-03-07T03:09:04.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:04 smithi145 bash[30906]: cluster 2024-03-07T03:09:03.380475+0000 mon.a (mon.0) 836 : cluster 0 osdmap e71: 8 total, 8 up, 8 in 2024-03-07T03:09:04.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:04 smithi189 bash[23614]: cluster 2024-03-07T03:09:02.407774+0000 mgr.y (mgr.24302) 130 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.4 KiB/s wr, 5 op/s 2024-03-07T03:09:04.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:04 smithi189 bash[23614]: audit 2024-03-07T03:09:03.379842+0000 mon.a (mon.0) 833 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-03-07T03:09:04.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:04 smithi189 bash[23614]: audit 2024-03-07T03:09:03.380100+0000 mon.a (mon.0) 834 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-03-07T03:09:04.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:04 smithi189 bash[23614]: audit 2024-03-07T03:09:03.380278+0000 mon.a (mon.0) 835 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-03-07T03:09:04.834 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:04 smithi189 bash[23614]: cluster 2024-03-07T03:09:03.380475+0000 mon.a (mon.0) 836 : cluster 0 osdmap e71: 8 total, 8 up, 8 in 2024-03-07T03:09:05.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:05 smithi145 bash[21339]: cluster 2024-03-07T03:09:04.391727+0000 mon.a (mon.0) 837 : cluster 0 osdmap e72: 8 total, 8 up, 8 in 2024-03-07T03:09:05.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:05 smithi145 bash[30906]: cluster 2024-03-07T03:09:04.391727+0000 mon.a (mon.0) 837 : cluster 0 osdmap e72: 8 total, 8 up, 8 in 2024-03-07T03:09:05.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:05 smithi189 bash[23614]: cluster 2024-03-07T03:09:04.391727+0000 mon.a (mon.0) 837 : cluster 0 osdmap e72: 8 total, 8 up, 8 in 2024-03-07T03:09:06.041 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:06.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:06 smithi145 bash[21339]: cluster 2024-03-07T03:09:04.408424+0000 mgr.y (mgr.24302) 131 : cluster 0 pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 2.0 KiB/s wr, 17 op/s 2024-03-07T03:09:06.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:06 smithi145 bash[30906]: cluster 2024-03-07T03:09:04.408424+0000 mgr.y (mgr.24302) 131 : cluster 0 pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 2.0 KiB/s wr, 17 op/s 2024-03-07T03:09:06.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:06 smithi189 bash[23614]: cluster 2024-03-07T03:09:04.408424+0000 mgr.y (mgr.24302) 131 : cluster 0 pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 2.0 KiB/s wr, 17 op/s 2024-03-07T03:09:07.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:07 smithi145 bash[21339]: cluster 2024-03-07T03:09:06.409038+0000 mgr.y (mgr.24302) 132 : cluster 0 pgmap v107: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 1.7 KiB/s wr, 15 op/s 2024-03-07T03:09:07.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:07 smithi145 bash[30906]: cluster 2024-03-07T03:09:06.409038+0000 mgr.y (mgr.24302) 132 : cluster 0 pgmap v107: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 1.7 KiB/s wr, 15 op/s 2024-03-07T03:09:07.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:07 smithi189 bash[23614]: cluster 2024-03-07T03:09:06.409038+0000 mgr.y (mgr.24302) 132 : cluster 0 pgmap v107: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 1.7 KiB/s wr, 15 op/s 2024-03-07T03:09:10.188 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-03-07T03:09:10.197 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-07T03:09:10.197 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-07T03:09:10.197 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph mgr dump --format=json 2024-03-07T03:09:10.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:10 smithi145 bash[21339]: cluster 2024-03-07T03:09:08.409704+0000 mgr.y (mgr.24302) 133 : cluster 0 pgmap v108: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-03-07T03:09:10.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:10 smithi145 bash[30906]: cluster 2024-03-07T03:09:08.409704+0000 mgr.y (mgr.24302) 133 : cluster 0 pgmap v108: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-03-07T03:09:10.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:10 smithi189 bash[23614]: cluster 2024-03-07T03:09:08.409704+0000 mgr.y (mgr.24302) 133 : cluster 0 pgmap v108: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-03-07T03:09:12.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:12 smithi145 bash[21339]: cluster 2024-03-07T03:09:10.410799+0000 mgr.y (mgr.24302) 134 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-03-07T03:09:12.814 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:09:12 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:09:12] "GET /metrics HTTP/1.1" 200 33767 "" "Prometheus/2.43.0" 2024-03-07T03:09:12.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:12 smithi145 bash[30906]: cluster 2024-03-07T03:09:10.410799+0000 mgr.y (mgr.24302) 134 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-03-07T03:09:12.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:12 smithi189 bash[23614]: cluster 2024-03-07T03:09:10.410799+0000 mgr.y (mgr.24302) 134 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-03-07T03:09:14.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:14 smithi145 bash[21339]: cluster 2024-03-07T03:09:12.411390+0000 mgr.y (mgr.24302) 135 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-03-07T03:09:14.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:14 smithi145 bash[30906]: cluster 2024-03-07T03:09:12.411390+0000 mgr.y (mgr.24302) 135 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-03-07T03:09:14.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:14 smithi189 bash[23614]: cluster 2024-03-07T03:09:12.411390+0000 mgr.y (mgr.24302) 135 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 0 B/s wr, 7 op/s 2024-03-07T03:09:14.934 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:16.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:16 smithi145 bash[21339]: cluster 2024-03-07T03:09:14.412489+0000 mgr.y (mgr.24302) 136 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:16.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:16 smithi145 bash[30906]: cluster 2024-03-07T03:09:14.412489+0000 mgr.y (mgr.24302) 136 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:16.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:16 smithi189 bash[23614]: cluster 2024-03-07T03:09:14.412489+0000 mgr.y (mgr.24302) 136 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:18.200 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:09:18.466 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:18 smithi145 bash[21339]: cluster 2024-03-07T03:09:16.413106+0000 mgr.y (mgr.24302) 137 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:18.466 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:18 smithi145 bash[21339]: audit 2024-03-07T03:09:17.450385+0000 mon.c (mon.2) 100 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:18.466 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:18 smithi145 bash[21339]: audit 2024-03-07T03:09:18.198443+0000 mon.a (mon.0) 838 : audit 0 from='client.? 172.21.15.145:0/2324516676' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:09:18.466 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:18 smithi145 bash[30906]: cluster 2024-03-07T03:09:16.413106+0000 mgr.y (mgr.24302) 137 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:18.467 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:18 smithi145 bash[30906]: audit 2024-03-07T03:09:17.450385+0000 mon.c (mon.2) 100 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:18.467 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:18 smithi145 bash[30906]: audit 2024-03-07T03:09:18.198443+0000 mon.a (mon.0) 838 : audit 0 from='client.? 172.21.15.145:0/2324516676' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:09:18.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:18 smithi189 bash[23614]: cluster 2024-03-07T03:09:16.413106+0000 mgr.y (mgr.24302) 137 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:18.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:18 smithi189 bash[23614]: audit 2024-03-07T03:09:17.450385+0000 mon.c (mon.2) 100 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:18.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:18 smithi189 bash[23614]: audit 2024-03-07T03:09:18.198443+0000 mon.a (mon.0) 838 : audit 0 from='client.? 172.21.15.145:0/2324516676' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:09:19.017 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":20,"active_gid":24302,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6800","nonce":1684821814},{"type":"v1","addr":"172.21.15.145:6801","nonce":1684821814}]},"active_addr":"172.21.15.145:6801/1684821814","active_change":"2024-03-07T03:06:02.331328+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24304,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.145:8443/","prometheus":"http://172.21.15.145:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":3349297031}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":1139507458}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":1420640097}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":4002816623}]}]} 2024-03-07T03:09:19.021 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-07T03:09:19.021 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-07T03:09:19.022 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:09:20.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:20 smithi145 bash[21339]: cluster 2024-03-07T03:09:18.413751+0000 mgr.y (mgr.24302) 138 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:20.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:20 smithi145 bash[30906]: cluster 2024-03-07T03:09:18.413751+0000 mgr.y (mgr.24302) 138 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:20.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:20 smithi189 bash[23614]: cluster 2024-03-07T03:09:18.413751+0000 mgr.y (mgr.24302) 138 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:22.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:22 smithi145 bash[21339]: cluster 2024-03-07T03:09:20.414880+0000 mgr.y (mgr.24302) 139 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:22.815 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:09:22 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:09:22] "GET /metrics HTTP/1.1" 200 33769 "" "Prometheus/2.43.0" 2024-03-07T03:09:22.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:22 smithi145 bash[30906]: cluster 2024-03-07T03:09:20.414880+0000 mgr.y (mgr.24302) 139 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:22.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:22 smithi189 bash[23614]: cluster 2024-03-07T03:09:20.414880+0000 mgr.y (mgr.24302) 139 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:23.766 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:24.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:24 smithi145 bash[21339]: cluster 2024-03-07T03:09:22.415502+0000 mgr.y (mgr.24302) 140 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:24.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:24 smithi145 bash[30906]: cluster 2024-03-07T03:09:22.415502+0000 mgr.y (mgr.24302) 140 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:24.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:24 smithi189 bash[23614]: cluster 2024-03-07T03:09:22.415502+0000 mgr.y (mgr.24302) 140 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:26.673 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:26 smithi145 bash[21339]: cluster 2024-03-07T03:09:24.416644+0000 mgr.y (mgr.24302) 141 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:26.673 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:26 smithi145 bash[30906]: cluster 2024-03-07T03:09:24.416644+0000 mgr.y (mgr.24302) 141 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:26.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:26 smithi189 bash[23614]: cluster 2024-03-07T03:09:24.416644+0000 mgr.y (mgr.24302) 141 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:27.073 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:09:27.073 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":72,"fsid":"8947a0f8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:55:42.039360+0000","modified":"2024-03-07T03:09:04.374824+0000","last_up_change":"2024-03-07T03:05:04.610834+0000","last_in_change":"2024-03-07T03:04:39.932769+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:00:59.846304+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-07T03:06:41.584515+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-07T03:08:48.151945+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-07T03:08:48.827708+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"242c63ee-0417-4ebf-bd71-6239f5d3634f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6803","nonce":1367316946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6805","nonce":1367316946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6809","nonce":1367316946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6807","nonce":1367316946}]},"public_addr":"172.21.15.145:6803/1367316946","cluster_addr":"172.21.15.145:6805/1367316946","heartbeat_back_addr":"172.21.15.145:6809/1367316946","heartbeat_front_addr":"172.21.15.145:6807/1367316946","state":["exists","up"]},{"osd":1,"uuid":"3c2818da-5d32-4f7a-8a63-8d7d679961e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6811","nonce":1873038359}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6813","nonce":1873038359}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6817","nonce":1873038359}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6815","nonce":1873038359}]},"public_addr":"172.21.15.145:6811/1873038359","cluster_addr":"172.21.15.145:6813/1873038359","heartbeat_back_addr":"172.21.15.145:6817/1873038359","heartbeat_front_addr":"172.21.15.145:6815/1873038359","state":["exists","up"]},{"osd":2,"uuid":"905ec219-dd9c-4f30-bdab-ed78d1405c11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6819","nonce":462216015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6821","nonce":462216015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6825","nonce":462216015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6823","nonce":462216015}]},"public_addr":"172.21.15.145:6819/462216015","cluster_addr":"172.21.15.145:6821/462216015","heartbeat_back_addr":"172.21.15.145:6825/462216015","heartbeat_front_addr":"172.21.15.145:6823/462216015","state":["exists","up"]},{"osd":3,"uuid":"5f466751-f5a7-4d18-87ab-96251b77b663","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6827","nonce":2306759084}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6829","nonce":2306759084}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6833","nonce":2306759084}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6831","nonce":2306759084}]},"public_addr":"172.21.15.145:6827/2306759084","cluster_addr":"172.21.15.145:6829/2306759084","heartbeat_back_addr":"172.21.15.145:6833/2306759084","heartbeat_front_addr":"172.21.15.145:6831/2306759084","state":["exists","up"]},{"osd":4,"uuid":"633f56af-3ff8-4954-97b4-bc8122bb10ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6801","nonce":3152992669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6803","nonce":3152992669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6807","nonce":3152992669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6805","nonce":3152992669}]},"public_addr":"172.21.15.189:6801/3152992669","cluster_addr":"172.21.15.189:6803/3152992669","heartbeat_back_addr":"172.21.15.189:6807/3152992669","heartbeat_front_addr":"172.21.15.189:6805/3152992669","state":["exists","up"]},{"osd":5,"uuid":"531d5127-60fe-4669-ab7d-fe4f5e5753cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6809","nonce":2096444201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6811","nonce":2096444201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6815","nonce":2096444201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6813","nonce":2096444201}]},"public_addr":"172.21.15.189:6809/2096444201","cluster_addr":"172.21.15.189:6811/2096444201","heartbeat_back_addr":"172.21.15.189:6815/2096444201","heartbeat_front_addr":"172.21.15.189:6813/2096444201","state":["exists","up"]},{"osd":6,"uuid":"8f2b456e-712e-45f1-8d84-0fdc4a6d860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6817","nonce":4219543665}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6819","nonce":4219543665}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6823","nonce":4219543665}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6821","nonce":4219543665}]},"public_addr":"172.21.15.189:6817/4219543665","cluster_addr":"172.21.15.189:6819/4219543665","heartbeat_back_addr":"172.21.15.189:6823/4219543665","heartbeat_front_addr":"172.21.15.189:6821/4219543665","state":["exists","up"]},{"osd":7,"uuid":"24dc7eba-4cd4-414d-93d2-598f5f2e655c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6825","nonce":452348965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6827","nonce":452348965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6831","nonce":452348965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6829","nonce":452348965}]},"public_addr":"172.21.15.189:6825/452348965","cluster_addr":"172.21.15.189:6827/452348965","heartbeat_back_addr":"172.21.15.189:6831/452348965","heartbeat_front_addr":"172.21.15.189:6829/452348965","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T02:59:11.112897+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:03.501748+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:56.635004+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:01:50.372735+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:38.150721+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:25.965239+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:14.345171+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:03.479901+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.145:6835/3433351665":"2024-03-08T03:08:57.328270+0000","172.21.15.145:6834/3433351665":"2024-03-08T03:08:57.328270+0000","172.21.15.145:6800/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/2414597158":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/2696605014":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:6800/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/1978862755":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/2236140381":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/3488231986":"2024-03-08T03:06:02.331013+0000","172.21.15.145:6801/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/321164949":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/131277059":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/2939737297":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6800/2891889777":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3852118874":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3299433409":"2024-03-08T03:06:02.331013+0000","172.21.15.145:6801/2891889777":"2024-03-08T03:06:02.331013+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:09:27.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:27 smithi145 bash[30906]: audit 2024-03-07T03:09:27.079901+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.145:0/2752431427' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:09:27.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:27 smithi145 bash[21339]: audit 2024-03-07T03:09:27.079901+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.145:0/2752431427' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:09:27.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:27 smithi189 bash[23614]: audit 2024-03-07T03:09:27.079901+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.145:0/2752431427' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:09:27.856 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-07T03:09:27.856 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:09:28.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:28 smithi145 bash[21339]: cluster 2024-03-07T03:09:26.417307+0000 mgr.y (mgr.24302) 142 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:28.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:28 smithi145 bash[30906]: cluster 2024-03-07T03:09:26.417307+0000 mgr.y (mgr.24302) 142 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:28.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:28 smithi189 bash[23614]: cluster 2024-03-07T03:09:26.417307+0000 mgr.y (mgr.24302) 142 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:29.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:29 smithi145 bash[21339]: cluster 2024-03-07T03:09:28.417886+0000 mgr.y (mgr.24302) 143 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:29.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:29 smithi145 bash[30906]: cluster 2024-03-07T03:09:28.417886+0000 mgr.y (mgr.24302) 143 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:29.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:29 smithi189 bash[23614]: cluster 2024-03-07T03:09:28.417886+0000 mgr.y (mgr.24302) 143 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:31.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:31 smithi145 bash[21339]: cluster 2024-03-07T03:09:30.418987+0000 mgr.y (mgr.24302) 144 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:31.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:31 smithi145 bash[30906]: cluster 2024-03-07T03:09:30.418987+0000 mgr.y (mgr.24302) 144 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:31.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:31 smithi189 bash[23614]: cluster 2024-03-07T03:09:30.418987+0000 mgr.y (mgr.24302) 144 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:32.590 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:32.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:32 smithi145 bash[21339]: audit 2024-03-07T03:09:32.450584+0000 mon.c (mon.2) 102 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:32.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:32 smithi145 bash[30906]: audit 2024-03-07T03:09:32.450584+0000 mon.c (mon.2) 102 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:32.814 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:09:32 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:09:32] "GET /metrics HTTP/1.1" 200 33769 "" "Prometheus/2.43.0" 2024-03-07T03:09:32.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:32 smithi189 bash[23614]: audit 2024-03-07T03:09:32.450584+0000 mon.c (mon.2) 102 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:34.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:34 smithi145 bash[21339]: cluster 2024-03-07T03:09:32.419620+0000 mgr.y (mgr.24302) 145 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:34.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:34 smithi145 bash[30906]: cluster 2024-03-07T03:09:32.419620+0000 mgr.y (mgr.24302) 145 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:34.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:34 smithi189 bash[23614]: cluster 2024-03-07T03:09:32.419620+0000 mgr.y (mgr.24302) 145 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:35.900 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:09:35.900 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":72,"fsid":"8947a0f8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:55:42.039360+0000","modified":"2024-03-07T03:09:04.374824+0000","last_up_change":"2024-03-07T03:05:04.610834+0000","last_in_change":"2024-03-07T03:04:39.932769+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:00:59.846304+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-07T03:06:41.584515+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-07T03:08:48.151945+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-07T03:08:48.827708+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"242c63ee-0417-4ebf-bd71-6239f5d3634f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6803","nonce":1367316946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6805","nonce":1367316946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6809","nonce":1367316946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":1367316946},{"type":"v1","addr":"172.21.15.145:6807","nonce":1367316946}]},"public_addr":"172.21.15.145:6803/1367316946","cluster_addr":"172.21.15.145:6805/1367316946","heartbeat_back_addr":"172.21.15.145:6809/1367316946","heartbeat_front_addr":"172.21.15.145:6807/1367316946","state":["exists","up"]},{"osd":1,"uuid":"3c2818da-5d32-4f7a-8a63-8d7d679961e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6811","nonce":1873038359}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6813","nonce":1873038359}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6817","nonce":1873038359}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":1873038359},{"type":"v1","addr":"172.21.15.145:6815","nonce":1873038359}]},"public_addr":"172.21.15.145:6811/1873038359","cluster_addr":"172.21.15.145:6813/1873038359","heartbeat_back_addr":"172.21.15.145:6817/1873038359","heartbeat_front_addr":"172.21.15.145:6815/1873038359","state":["exists","up"]},{"osd":2,"uuid":"905ec219-dd9c-4f30-bdab-ed78d1405c11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6819","nonce":462216015}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6821","nonce":462216015}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6825","nonce":462216015}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":462216015},{"type":"v1","addr":"172.21.15.145:6823","nonce":462216015}]},"public_addr":"172.21.15.145:6819/462216015","cluster_addr":"172.21.15.145:6821/462216015","heartbeat_back_addr":"172.21.15.145:6825/462216015","heartbeat_front_addr":"172.21.15.145:6823/462216015","state":["exists","up"]},{"osd":3,"uuid":"5f466751-f5a7-4d18-87ab-96251b77b663","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6827","nonce":2306759084}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6829","nonce":2306759084}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6833","nonce":2306759084}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2306759084},{"type":"v1","addr":"172.21.15.145:6831","nonce":2306759084}]},"public_addr":"172.21.15.145:6827/2306759084","cluster_addr":"172.21.15.145:6829/2306759084","heartbeat_back_addr":"172.21.15.145:6833/2306759084","heartbeat_front_addr":"172.21.15.145:6831/2306759084","state":["exists","up"]},{"osd":4,"uuid":"633f56af-3ff8-4954-97b4-bc8122bb10ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6801","nonce":3152992669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6803","nonce":3152992669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6807","nonce":3152992669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3152992669},{"type":"v1","addr":"172.21.15.189:6805","nonce":3152992669}]},"public_addr":"172.21.15.189:6801/3152992669","cluster_addr":"172.21.15.189:6803/3152992669","heartbeat_back_addr":"172.21.15.189:6807/3152992669","heartbeat_front_addr":"172.21.15.189:6805/3152992669","state":["exists","up"]},{"osd":5,"uuid":"531d5127-60fe-4669-ab7d-fe4f5e5753cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6809","nonce":2096444201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6811","nonce":2096444201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6815","nonce":2096444201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6813","nonce":2096444201}]},"public_addr":"172.21.15.189:6809/2096444201","cluster_addr":"172.21.15.189:6811/2096444201","heartbeat_back_addr":"172.21.15.189:6815/2096444201","heartbeat_front_addr":"172.21.15.189:6813/2096444201","state":["exists","up"]},{"osd":6,"uuid":"8f2b456e-712e-45f1-8d84-0fdc4a6d860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6817","nonce":4219543665}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6819","nonce":4219543665}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6823","nonce":4219543665}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6821","nonce":4219543665}]},"public_addr":"172.21.15.189:6817/4219543665","cluster_addr":"172.21.15.189:6819/4219543665","heartbeat_back_addr":"172.21.15.189:6823/4219543665","heartbeat_front_addr":"172.21.15.189:6821/4219543665","state":["exists","up"]},{"osd":7,"uuid":"24dc7eba-4cd4-414d-93d2-598f5f2e655c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6825","nonce":452348965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6827","nonce":452348965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6831","nonce":452348965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6829","nonce":452348965}]},"public_addr":"172.21.15.189:6825/452348965","cluster_addr":"172.21.15.189:6827/452348965","heartbeat_back_addr":"172.21.15.189:6831/452348965","heartbeat_front_addr":"172.21.15.189:6829/452348965","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T02:59:11.112897+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:03.501748+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:56.635004+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:01:50.372735+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:38.150721+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:25.965239+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:14.345171+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:03.479901+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.145:6835/3433351665":"2024-03-08T03:08:57.328270+0000","172.21.15.145:6834/3433351665":"2024-03-08T03:08:57.328270+0000","172.21.15.145:6800/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/2414597158":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/2696605014":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:6800/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/1978862755":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/2236140381":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/3488231986":"2024-03-08T03:06:02.331013+0000","172.21.15.145:6801/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/321164949":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/131277059":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/2939737297":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6800/2891889777":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3852118874":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3299433409":"2024-03-08T03:06:02.331013+0000","172.21.15.145:6801/2891889777":"2024-03-08T03:06:02.331013+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:09:36.664 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:36 smithi145 bash[21339]: cluster 2024-03-07T03:09:34.420764+0000 mgr.y (mgr.24302) 146 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:36.664 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:36 smithi145 bash[21339]: audit 2024-03-07T03:09:35.905148+0000 mon.c (mon.2) 103 : audit 0 from='client.? 172.21.15.145:0/4104687428' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:09:36.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:36 smithi145 bash[30906]: cluster 2024-03-07T03:09:34.420764+0000 mgr.y (mgr.24302) 146 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:36.665 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:36 smithi145 bash[30906]: audit 2024-03-07T03:09:35.905148+0000 mon.c (mon.2) 103 : audit 0 from='client.? 172.21.15.145:0/4104687428' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:09:36.666 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-07T03:09:36.666 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-07T03:09:36.667 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-07T03:09:36.667 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-07T03:09:36.668 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-07T03:09:36.668 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-07T03:09:36.669 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-07T03:09:36.669 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-07T03:09:36.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:36 smithi189 bash[23614]: cluster 2024-03-07T03:09:34.420764+0000 mgr.y (mgr.24302) 146 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:36.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:36 smithi189 bash[23614]: audit 2024-03-07T03:09:35.905148+0000 mon.c (mon.2) 103 : audit 0 from='client.? 172.21.15.145:0/4104687428' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:09:38.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:38 smithi145 bash[21339]: cluster 2024-03-07T03:09:36.421440+0000 mgr.y (mgr.24302) 147 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:38.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:38 smithi145 bash[30906]: cluster 2024-03-07T03:09:36.421440+0000 mgr.y (mgr.24302) 147 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:38.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:38 smithi189 bash[23614]: cluster 2024-03-07T03:09:36.421440+0000 mgr.y (mgr.24302) 147 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:40.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:40 smithi145 bash[21339]: cluster 2024-03-07T03:09:38.422230+0000 mgr.y (mgr.24302) 148 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:40.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:40 smithi145 bash[30906]: cluster 2024-03-07T03:09:38.422230+0000 mgr.y (mgr.24302) 148 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:40.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:40 smithi189 bash[23614]: cluster 2024-03-07T03:09:38.422230+0000 mgr.y (mgr.24302) 148 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:41.507 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:41.508 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:41.511 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:41.511 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:41.512 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:41.512 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:41.514 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:41.514 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:42.813 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:09:42 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:09:42] "GET /metrics HTTP/1.1" 200 33770 "" "Prometheus/2.43.0" 2024-03-07T03:09:42.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:42 smithi145 bash[21339]: cluster 2024-03-07T03:09:40.423411+0000 mgr.y (mgr.24302) 149 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:42.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:42 smithi145 bash[30906]: cluster 2024-03-07T03:09:40.423411+0000 mgr.y (mgr.24302) 149 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:42.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:42 smithi189 bash[23614]: cluster 2024-03-07T03:09:40.423411+0000 mgr.y (mgr.24302) 149 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:44.506 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:44 smithi145 bash[21339]: cluster 2024-03-07T03:09:42.424010+0000 mgr.y (mgr.24302) 150 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:44.506 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:44 smithi145 bash[30906]: cluster 2024-03-07T03:09:42.424010+0000 mgr.y (mgr.24302) 150 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:44.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:44 smithi189 bash[23614]: cluster 2024-03-07T03:09:42.424010+0000 mgr.y (mgr.24302) 150 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:46.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:46 smithi145 bash[21339]: cluster 2024-03-07T03:09:44.424667+0000 mgr.y (mgr.24302) 151 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:46.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:46 smithi145 bash[30906]: cluster 2024-03-07T03:09:44.424667+0000 mgr.y (mgr.24302) 151 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:46.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:46 smithi189 bash[23614]: cluster 2024-03-07T03:09:44.424667+0000 mgr.y (mgr.24302) 151 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:48.542 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:48 smithi145 bash[21339]: cluster 2024-03-07T03:09:46.425276+0000 mgr.y (mgr.24302) 152 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:48.542 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:48 smithi145 bash[21339]: audit 2024-03-07T03:09:47.451340+0000 mon.c (mon.2) 104 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:48.542 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:48 smithi145 bash[30906]: cluster 2024-03-07T03:09:46.425276+0000 mgr.y (mgr.24302) 152 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:48.542 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:48 smithi145 bash[30906]: audit 2024-03-07T03:09:47.451340+0000 mon.c (mon.2) 104 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:48.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:48 smithi189 bash[23614]: cluster 2024-03-07T03:09:46.425276+0000 mgr.y (mgr.24302) 152 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:48.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:48 smithi189 bash[23614]: audit 2024-03-07T03:09:47.451340+0000 mon.c (mon.2) 104 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:50.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:50 smithi145 bash[21339]: cluster 2024-03-07T03:09:48.425697+0000 mgr.y (mgr.24302) 153 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:50.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:50 smithi145 bash[30906]: cluster 2024-03-07T03:09:48.425697+0000 mgr.y (mgr.24302) 153 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:50.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:50 smithi189 bash[23614]: cluster 2024-03-07T03:09:48.425697+0000 mgr.y (mgr.24302) 153 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:51.778 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:51 smithi145 bash[21339]: cluster 2024-03-07T03:09:50.426745+0000 mgr.y (mgr.24302) 154 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:51.779 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:51 smithi145 bash[30906]: cluster 2024-03-07T03:09:50.426745+0000 mgr.y (mgr.24302) 154 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:51.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:51 smithi189 bash[23614]: cluster 2024-03-07T03:09:50.426745+0000 mgr.y (mgr.24302) 154 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:52.563 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:09:52 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:09:52] "GET /metrics HTTP/1.1" 200 33768 "" "Prometheus/2.43.0" 2024-03-07T03:09:54.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:54 smithi145 bash[30906]: cluster 2024-03-07T03:09:52.427313+0000 mgr.y (mgr.24302) 155 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:54.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:54 smithi145 bash[21339]: cluster 2024-03-07T03:09:52.427313+0000 mgr.y (mgr.24302) 155 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:54.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:54 smithi189 bash[23614]: cluster 2024-03-07T03:09:52.427313+0000 mgr.y (mgr.24302) 155 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:56.639 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:56 smithi145 bash[21339]: cluster 2024-03-07T03:09:54.428093+0000 mgr.y (mgr.24302) 156 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:56.639 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:56 smithi145 bash[30906]: cluster 2024-03-07T03:09:54.428093+0000 mgr.y (mgr.24302) 156 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:56.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:56 smithi189 bash[23614]: cluster 2024-03-07T03:09:54.428093+0000 mgr.y (mgr.24302) 156 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:57.631 INFO:teuthology.orchestra.run.smithi145.stdout:133143986265 2024-03-07T03:09:57.632 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-07T03:09:58.641 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:09:58 smithi145 bash[21339]: cluster 2024-03-07T03:09:56.428874+0000 mgr.y (mgr.24302) 157 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:58.642 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:09:58 smithi145 bash[30906]: cluster 2024-03-07T03:09:56.428874+0000 mgr.y (mgr.24302) 157 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:58.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:09:58 smithi189 bash[23614]: cluster 2024-03-07T03:09:56.428874+0000 mgr.y (mgr.24302) 157 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:00.365 INFO:teuthology.orchestra.run.smithi145.stdout:201863462983 2024-03-07T03:10:00.365 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-07T03:10:00.454 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:00 smithi145 bash[21339]: cluster 2024-03-07T03:09:58.429623+0000 mgr.y (mgr.24302) 158 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:00.454 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:00 smithi145 bash[21339]: cluster 2024-03-07T03:10:00.000140+0000 mon.a (mon.0) 839 : cluster 1 overall HEALTH_OK 2024-03-07T03:10:00.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:00 smithi145 bash[30906]: cluster 2024-03-07T03:09:58.429623+0000 mgr.y (mgr.24302) 158 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:00.454 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:00 smithi145 bash[30906]: cluster 2024-03-07T03:10:00.000140+0000 mon.a (mon.0) 839 : cluster 1 overall HEALTH_OK 2024-03-07T03:10:00.552 INFO:teuthology.orchestra.run.smithi145.stdout:55834574969 2024-03-07T03:10:00.552 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-07T03:10:00.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:00 smithi189 bash[23614]: cluster 2024-03-07T03:09:58.429623+0000 mgr.y (mgr.24302) 158 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:00.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:00 smithi189 bash[23614]: cluster 2024-03-07T03:10:00.000140+0000 mon.a (mon.0) 839 : cluster 1 overall HEALTH_OK 2024-03-07T03:10:02.420 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:10:02 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:10:02] "GET /metrics HTTP/1.1" 200 33768 "" "Prometheus/2.43.0" 2024-03-07T03:10:02.420 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:02 smithi145 bash[21339]: cluster 2024-03-07T03:10:00.430284+0000 mgr.y (mgr.24302) 159 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:02.420 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:02 smithi145 bash[21339]: audit 2024-03-07T03:10:01.850456+0000 mon.c (mon.2) 105 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:10:02.425 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:02 smithi145 bash[30906]: cluster 2024-03-07T03:10:00.430284+0000 mgr.y (mgr.24302) 159 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:02.425 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:02 smithi145 bash[30906]: audit 2024-03-07T03:10:01.850456+0000 mon.c (mon.2) 105 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:10:02.444 INFO:teuthology.orchestra.run.smithi145.stdout:111669149795 2024-03-07T03:10:02.444 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-07T03:10:02.445 INFO:teuthology.orchestra.run.smithi145.stdout:236223201342 2024-03-07T03:10:02.446 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-07T03:10:02.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:02 smithi189 bash[23614]: cluster 2024-03-07T03:10:00.430284+0000 mgr.y (mgr.24302) 159 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:02.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:02 smithi189 bash[23614]: audit 2024-03-07T03:10:01.850456+0000 mon.c (mon.2) 105 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:10:03.450 INFO:teuthology.orchestra.run.smithi145.stdout:77309411440 2024-03-07T03:10:03.450 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-07T03:10:03.715 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:03 smithi145 bash[21339]: audit 2024-03-07T03:10:02.429574+0000 mon.c (mon.2) 106 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:10:03.715 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:03 smithi145 bash[21339]: audit 2024-03-07T03:10:02.434826+0000 mon.c (mon.2) 107 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:10:03.716 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:03 smithi145 bash[21339]: audit 2024-03-07T03:10:02.442493+0000 mon.a (mon.0) 840 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:10:03.716 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:03 smithi145 bash[21339]: audit 2024-03-07T03:10:02.450961+0000 mon.c (mon.2) 108 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:03.716 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:03 smithi145 bash[30906]: audit 2024-03-07T03:10:02.429574+0000 mon.c (mon.2) 106 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:10:03.716 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:03 smithi145 bash[30906]: audit 2024-03-07T03:10:02.434826+0000 mon.c (mon.2) 107 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:10:03.716 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:03 smithi145 bash[30906]: audit 2024-03-07T03:10:02.442493+0000 mon.a (mon.0) 840 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:10:03.716 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:03 smithi145 bash[30906]: audit 2024-03-07T03:10:02.450961+0000 mon.c (mon.2) 108 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:03.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:03 smithi189 bash[23614]: audit 2024-03-07T03:10:02.429574+0000 mon.c (mon.2) 106 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:10:03.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:03 smithi189 bash[23614]: audit 2024-03-07T03:10:02.434826+0000 mon.c (mon.2) 107 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:10:03.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:03 smithi189 bash[23614]: audit 2024-03-07T03:10:02.442493+0000 mon.a (mon.0) 840 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:10:03.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:03 smithi189 bash[23614]: audit 2024-03-07T03:10:02.450961+0000 mon.c (mon.2) 108 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:03.972 INFO:teuthology.orchestra.run.smithi145.stdout:167503724625 2024-03-07T03:10:03.972 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-07T03:10:04.480 INFO:teuthology.orchestra.run.smithi145.stdout:34359738500 2024-03-07T03:10:04.480 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-07T03:10:04.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:04 smithi145 bash[21339]: cluster 2024-03-07T03:10:02.430739+0000 mgr.y (mgr.24302) 160 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:04.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:04 smithi145 bash[30906]: cluster 2024-03-07T03:10:02.430739+0000 mgr.y (mgr.24302) 160 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:04.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:04 smithi189 bash[23614]: cluster 2024-03-07T03:10:02.430739+0000 mgr.y (mgr.24302) 160 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:05.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:05 smithi145 bash[21339]: cluster 2024-03-07T03:10:04.431770+0000 mgr.y (mgr.24302) 161 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:05.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:05 smithi145 bash[30906]: cluster 2024-03-07T03:10:04.431770+0000 mgr.y (mgr.24302) 161 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:05.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:05 smithi189 bash[23614]: cluster 2024-03-07T03:10:04.431770+0000 mgr.y (mgr.24302) 161 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:07.462 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:07.462 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:07.463 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:07.463 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:07.463 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:07.464 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:07.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:07 smithi145 bash[21339]: cluster 2024-03-07T03:10:06.432409+0000 mgr.y (mgr.24302) 162 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:07.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:07 smithi145 bash[30906]: cluster 2024-03-07T03:10:06.432409+0000 mgr.y (mgr.24302) 162 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:07.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:07 smithi189 bash[23614]: cluster 2024-03-07T03:10:06.432409+0000 mgr.y (mgr.24302) 162 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:08.482 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:08.482 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:10.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:10 smithi145 bash[21339]: cluster 2024-03-07T03:10:08.433069+0000 mgr.y (mgr.24302) 163 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:10.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:10 smithi145 bash[30906]: cluster 2024-03-07T03:10:08.433069+0000 mgr.y (mgr.24302) 163 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:10.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:10 smithi189 bash[23614]: cluster 2024-03-07T03:10:08.433069+0000 mgr.y (mgr.24302) 163 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:12.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:12 smithi145 bash[21339]: cluster 2024-03-07T03:10:10.434177+0000 mgr.y (mgr.24302) 164 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:12.563 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:10:12 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:10:12] "GET /metrics HTTP/1.1" 200 33771 "" "Prometheus/2.43.0" 2024-03-07T03:10:12.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:12 smithi145 bash[30906]: cluster 2024-03-07T03:10:10.434177+0000 mgr.y (mgr.24302) 164 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:12.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:12 smithi189 bash[23614]: cluster 2024-03-07T03:10:10.434177+0000 mgr.y (mgr.24302) 164 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:14.511 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:14 smithi145 bash[21339]: cluster 2024-03-07T03:10:12.434667+0000 mgr.y (mgr.24302) 165 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:14.511 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:14 smithi145 bash[30906]: cluster 2024-03-07T03:10:12.434667+0000 mgr.y (mgr.24302) 165 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:14 smithi189 bash[23614]: cluster 2024-03-07T03:10:12.434667+0000 mgr.y (mgr.24302) 165 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:16.647 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:16 smithi145 bash[21339]: cluster 2024-03-07T03:10:14.435677+0000 mgr.y (mgr.24302) 166 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:16.647 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:16 smithi145 bash[30906]: cluster 2024-03-07T03:10:14.435677+0000 mgr.y (mgr.24302) 166 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:16.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:16 smithi189 bash[23614]: cluster 2024-03-07T03:10:14.435677+0000 mgr.y (mgr.24302) 166 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:17.717 INFO:teuthology.orchestra.run.smithi145.stdout:236223201345 2024-03-07T03:10:18.776 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:18 smithi145 bash[21339]: cluster 2024-03-07T03:10:16.436091+0000 mgr.y (mgr.24302) 167 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:18.776 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:18 smithi145 bash[21339]: audit 2024-03-07T03:10:17.450945+0000 mon.c (mon.2) 109 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:18.776 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:18 smithi145 bash[21339]: audit 2024-03-07T03:10:17.726065+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.145:0/2998845504' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:10:18.777 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:18 smithi145 bash[30906]: cluster 2024-03-07T03:10:16.436091+0000 mgr.y (mgr.24302) 167 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:18.777 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:18 smithi145 bash[30906]: audit 2024-03-07T03:10:17.450945+0000 mon.c (mon.2) 109 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:18.777 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:18 smithi145 bash[30906]: audit 2024-03-07T03:10:17.726065+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.145:0/2998845504' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:10:18.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:18 smithi189 bash[23614]: cluster 2024-03-07T03:10:16.436091+0000 mgr.y (mgr.24302) 167 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:18.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:18 smithi189 bash[23614]: audit 2024-03-07T03:10:17.450945+0000 mon.c (mon.2) 109 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:18.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:18 smithi189 bash[23614]: audit 2024-03-07T03:10:17.726065+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.145:0/2998845504' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:10:20.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:20 smithi145 bash[30906]: cluster 2024-03-07T03:10:18.436773+0000 mgr.y (mgr.24302) 168 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:20.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:20 smithi145 bash[21339]: cluster 2024-03-07T03:10:18.436773+0000 mgr.y (mgr.24302) 168 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:20.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:20 smithi189 bash[23614]: cluster 2024-03-07T03:10:18.436773+0000 mgr.y (mgr.24302) 168 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:21.932 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201342 got 236223201345 for osd.7 2024-03-07T03:10:21.932 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:22.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:22 smithi189 bash[23614]: cluster 2024-03-07T03:10:20.437940+0000 mgr.y (mgr.24302) 169 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:22.863 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:22 smithi145 bash[30906]: cluster 2024-03-07T03:10:20.437940+0000 mgr.y (mgr.24302) 169 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:22.864 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:22 smithi145 bash[21339]: cluster 2024-03-07T03:10:20.437940+0000 mgr.y (mgr.24302) 169 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:22.864 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:10:22 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:10:22] "GET /metrics HTTP/1.1" 200 33767 "" "Prometheus/2.43.0" 2024-03-07T03:10:22.949 INFO:teuthology.orchestra.run.smithi145.stdout:201863462988 2024-03-07T03:10:23.341 INFO:teuthology.orchestra.run.smithi145.stdout:111669149800 2024-03-07T03:10:23.661 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:23 smithi145 bash[21339]: audit 2024-03-07T03:10:22.958091+0000 mon.c (mon.2) 111 : audit 0 from='client.? 172.21.15.145:0/3217037099' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:10:23.661 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:23 smithi145 bash[21339]: audit 2024-03-07T03:10:23.350002+0000 mon.a (mon.0) 841 : audit 0 from='client.? 172.21.15.145:0/614815694' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:10:23.661 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:23 smithi145 bash[30906]: audit 2024-03-07T03:10:22.958091+0000 mon.c (mon.2) 111 : audit 0 from='client.? 172.21.15.145:0/3217037099' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:10:23.661 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:23 smithi145 bash[30906]: audit 2024-03-07T03:10:23.350002+0000 mon.a (mon.0) 841 : audit 0 from='client.? 172.21.15.145:0/614815694' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:10:23.700 INFO:teuthology.orchestra.run.smithi145.stdout:133143986271 2024-03-07T03:10:23.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:23 smithi189 bash[23614]: audit 2024-03-07T03:10:22.958091+0000 mon.c (mon.2) 111 : audit 0 from='client.? 172.21.15.145:0/3217037099' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:10:23.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:23 smithi189 bash[23614]: audit 2024-03-07T03:10:23.350002+0000 mon.a (mon.0) 841 : audit 0 from='client.? 172.21.15.145:0/614815694' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:10:24.027 INFO:teuthology.orchestra.run.smithi145.stdout:77309411444 2024-03-07T03:10:24.098 INFO:teuthology.orchestra.run.smithi145.stdout:34359738504 2024-03-07T03:10:24.794 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:24 smithi145 bash[21339]: cluster 2024-03-07T03:10:22.438576+0000 mgr.y (mgr.24302) 170 : cluster 0 pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:24.794 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:24 smithi145 bash[21339]: audit 2024-03-07T03:10:23.708981+0000 mon.a (mon.0) 842 : audit 0 from='client.? 172.21.15.145:0/2084214623' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:10:24.794 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:24 smithi145 bash[21339]: audit 2024-03-07T03:10:24.035813+0000 mon.a (mon.0) 843 : audit 0 from='client.? 172.21.15.145:0/3648239627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:10:24.795 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:24 smithi145 bash[21339]: audit 2024-03-07T03:10:24.107221+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.145:0/3012957062' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:10:24.795 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:24 smithi145 bash[30906]: cluster 2024-03-07T03:10:22.438576+0000 mgr.y (mgr.24302) 170 : cluster 0 pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:24.795 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:24 smithi145 bash[30906]: audit 2024-03-07T03:10:23.708981+0000 mon.a (mon.0) 842 : audit 0 from='client.? 172.21.15.145:0/2084214623' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:10:24.795 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:24 smithi145 bash[30906]: audit 2024-03-07T03:10:24.035813+0000 mon.a (mon.0) 843 : audit 0 from='client.? 172.21.15.145:0/3648239627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:10:24.795 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:24 smithi145 bash[30906]: audit 2024-03-07T03:10:24.107221+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.145:0/3012957062' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:10:24.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:24 smithi189 bash[23614]: cluster 2024-03-07T03:10:22.438576+0000 mgr.y (mgr.24302) 170 : cluster 0 pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:24.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:24 smithi189 bash[23614]: audit 2024-03-07T03:10:23.708981+0000 mon.a (mon.0) 842 : audit 0 from='client.? 172.21.15.145:0/2084214623' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:10:24.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:24 smithi189 bash[23614]: audit 2024-03-07T03:10:24.035813+0000 mon.a (mon.0) 843 : audit 0 from='client.? 172.21.15.145:0/3648239627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:10:24.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:24 smithi189 bash[23614]: audit 2024-03-07T03:10:24.107221+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.145:0/3012957062' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:10:25.199 INFO:teuthology.orchestra.run.smithi145.stdout:55834574974 2024-03-07T03:10:25.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:25 smithi145 bash[21339]: cluster 2024-03-07T03:10:24.439659+0000 mgr.y (mgr.24302) 171 : cluster 0 pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:25.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:25 smithi145 bash[21339]: audit 2024-03-07T03:10:25.208334+0000 mon.a (mon.0) 844 : audit 0 from='client.? 172.21.15.145:0/608567007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:10:25.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:25 smithi145 bash[30906]: cluster 2024-03-07T03:10:24.439659+0000 mgr.y (mgr.24302) 171 : cluster 0 pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:25.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:25 smithi145 bash[30906]: audit 2024-03-07T03:10:25.208334+0000 mon.a (mon.0) 844 : audit 0 from='client.? 172.21.15.145:0/608567007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:10:25.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:25 smithi189 bash[23614]: cluster 2024-03-07T03:10:24.439659+0000 mgr.y (mgr.24302) 171 : cluster 0 pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:25.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:25 smithi189 bash[23614]: audit 2024-03-07T03:10:25.208334+0000 mon.a (mon.0) 844 : audit 0 from='client.? 172.21.15.145:0/608567007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:10:25.946 INFO:teuthology.orchestra.run.smithi145.stdout:167503724630 2024-03-07T03:10:26.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:26 smithi145 bash[21339]: audit 2024-03-07T03:10:25.955795+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.145:0/1222486736' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:10:26.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:26 smithi145 bash[30906]: audit 2024-03-07T03:10:25.955795+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.145:0/1222486736' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:10:26.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:26 smithi189 bash[23614]: audit 2024-03-07T03:10:25.955795+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.145:0/1222486736' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:10:26.922 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462983 got 201863462988 for osd.6 2024-03-07T03:10:26.922 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:26.958 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149795 got 111669149800 for osd.3 2024-03-07T03:10:26.958 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:27.496 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:27 smithi145 bash[21339]: cluster 2024-03-07T03:10:26.440296+0000 mgr.y (mgr.24302) 172 : cluster 0 pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:27.496 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:27 smithi145 bash[30906]: cluster 2024-03-07T03:10:26.440296+0000 mgr.y (mgr.24302) 172 : cluster 0 pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:27.498 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986265 got 133143986271 for osd.4 2024-03-07T03:10:27.498 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:27.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:27 smithi189 bash[23614]: cluster 2024-03-07T03:10:26.440296+0000 mgr.y (mgr.24302) 172 : cluster 0 pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:27.967 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411440 got 77309411444 for osd.2 2024-03-07T03:10:27.968 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:28.331 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574969 got 55834574974 for osd.1 2024-03-07T03:10:28.331 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:29.294 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724625 got 167503724630 for osd.5 2024-03-07T03:10:29.294 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:29.311 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738500 got 34359738504 for osd.0 2024-03-07T03:10:29.311 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:29.311 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-07T03:10:29.311 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph pg dump --format=json 2024-03-07T03:10:30.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:30 smithi145 bash[21339]: cluster 2024-03-07T03:10:28.440899+0000 mgr.y (mgr.24302) 173 : cluster 0 pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:30.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:30 smithi145 bash[30906]: cluster 2024-03-07T03:10:28.440899+0000 mgr.y (mgr.24302) 173 : cluster 0 pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:30.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:30 smithi189 bash[23614]: cluster 2024-03-07T03:10:28.440899+0000 mgr.y (mgr.24302) 173 : cluster 0 pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:32.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:32 smithi145 bash[21339]: cluster 2024-03-07T03:10:30.441981+0000 mgr.y (mgr.24302) 174 : cluster 0 pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:32.813 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:10:32 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:10:32] "GET /metrics HTTP/1.1" 200 33767 "" "Prometheus/2.43.0" 2024-03-07T03:10:32.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:32 smithi145 bash[30906]: cluster 2024-03-07T03:10:30.441981+0000 mgr.y (mgr.24302) 174 : cluster 0 pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:32.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:32 smithi189 bash[23614]: cluster 2024-03-07T03:10:30.441981+0000 mgr.y (mgr.24302) 174 : cluster 0 pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:33.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:33 smithi145 bash[21339]: audit 2024-03-07T03:10:32.452006+0000 mon.c (mon.2) 114 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:33.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:33 smithi145 bash[30906]: audit 2024-03-07T03:10:32.452006+0000 mon.c (mon.2) 114 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:33.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:33 smithi189 bash[23614]: audit 2024-03-07T03:10:32.452006+0000 mon.c (mon.2) 114 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:34.102 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:34.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:34 smithi145 bash[21339]: cluster 2024-03-07T03:10:32.442593+0000 mgr.y (mgr.24302) 175 : cluster 0 pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:34.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:34 smithi145 bash[30906]: cluster 2024-03-07T03:10:32.442593+0000 mgr.y (mgr.24302) 175 : cluster 0 pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:34.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:34 smithi189 bash[23614]: cluster 2024-03-07T03:10:32.442593+0000 mgr.y (mgr.24302) 175 : cluster 0 pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:36.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:36 smithi145 bash[21339]: cluster 2024-03-07T03:10:34.443421+0000 mgr.y (mgr.24302) 176 : cluster 0 pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:36.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:36 smithi145 bash[30906]: cluster 2024-03-07T03:10:34.443421+0000 mgr.y (mgr.24302) 176 : cluster 0 pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:36 smithi189 bash[23614]: cluster 2024-03-07T03:10:34.443421+0000 mgr.y (mgr.24302) 176 : cluster 0 pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:37.348 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:10:37.352 INFO:teuthology.orchestra.run.smithi145.stderr:dumped all 2024-03-07T03:10:37.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:37 smithi145 bash[21339]: cluster 2024-03-07T03:10:36.444001+0000 mgr.y (mgr.24302) 177 : cluster 0 pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:37.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:37 smithi145 bash[21339]: audit 2024-03-07T03:10:37.347055+0000 mgr.y (mgr.24302) 178 : audit 0 from='client.24635 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:10:37.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:37 smithi145 bash[30906]: cluster 2024-03-07T03:10:36.444001+0000 mgr.y (mgr.24302) 177 : cluster 0 pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:37.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:37 smithi145 bash[30906]: audit 2024-03-07T03:10:37.347055+0000 mgr.y (mgr.24302) 178 : audit 0 from='client.24635 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:10:37.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:37 smithi189 bash[23614]: cluster 2024-03-07T03:10:36.444001+0000 mgr.y (mgr.24302) 177 : cluster 0 pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:37.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:37 smithi189 bash[23614]: audit 2024-03-07T03:10:37.347055+0000 mgr.y (mgr.24302) 178 : audit 0 from='client.24635 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:10:38.158 INFO:teuthology.orchestra.run.smithi145.stdout:{"pg_ready":true,"pg_map":{"version":152,"stamp":"2024-03-07T03:10:36.443671+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592730,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":289,"num_read_kb":386,"num_write":224,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173,"ondisk_log_size":173,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":223788,"kb_used_data":7600,"kb_used_omap":13,"kb_used_meta":216050,"kb_avail":749737428,"statfs":{"total":767960285184,"available":767731126272,"internally_reserved":0,"allocated":7782400,"data_stored":4372785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14248,"internal_metadata":221235288},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004803"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.387813+0000","last_change":"2024-03-07T03:08:49.451405+0000","last_active":"2024-03-07T03:09:03.387813+0000","last_peered":"2024-03-07T03:09:03.387813+0000","last_clean":"2024-03-07T03:09:03.387813+0000","last_became_active":"2024-03-07T03:08:49.450204+0000","last_became_peered":"2024-03-07T03:08:49.450204+0000","last_unstale":"2024-03-07T03:09:03.387813+0000","last_undegraded":"2024-03-07T03:09:03.387813+0000","last_fullsized":"2024-03-07T03:09:03.387813+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:20:33.165811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.390799+0000","last_change":"2024-03-07T03:08:49.428857+0000","last_active":"2024-03-07T03:09:03.390799+0000","last_peered":"2024-03-07T03:09:03.390799+0000","last_clean":"2024-03-07T03:09:03.390799+0000","last_became_active":"2024-03-07T03:08:49.426339+0000","last_became_peered":"2024-03-07T03:08:49.426339+0000","last_unstale":"2024-03-07T03:09:03.390799+0000","last_undegraded":"2024-03-07T03:09:03.390799+0000","last_fullsized":"2024-03-07T03:09:03.390799+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:04:00.929586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377229+0000","last_change":"2024-03-07T03:08:50.443295+0000","last_active":"2024-03-07T03:09:03.377229+0000","last_peered":"2024-03-07T03:09:03.377229+0000","last_clean":"2024-03-07T03:09:03.377229+0000","last_became_active":"2024-03-07T03:08:50.442492+0000","last_became_peered":"2024-03-07T03:08:50.442492+0000","last_unstale":"2024-03-07T03:09:03.377229+0000","last_undegraded":"2024-03-07T03:09:03.377229+0000","last_fullsized":"2024-03-07T03:09:03.377229+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:42:15.472843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377993+0000","last_change":"2024-03-07T03:08:49.420961+0000","last_active":"2024-03-07T03:09:03.377993+0000","last_peered":"2024-03-07T03:09:03.377993+0000","last_clean":"2024-03-07T03:09:03.377993+0000","last_became_active":"2024-03-07T03:08:49.420313+0000","last_became_peered":"2024-03-07T03:08:49.420313+0000","last_unstale":"2024-03-07T03:09:03.377993+0000","last_undegraded":"2024-03-07T03:09:03.377993+0000","last_fullsized":"2024-03-07T03:09:03.377993+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:13:19.978685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388468+0000","last_change":"2024-03-07T03:08:50.124290+0000","last_active":"2024-03-07T03:09:03.388468+0000","last_peered":"2024-03-07T03:09:03.388468+0000","last_clean":"2024-03-07T03:09:03.388468+0000","last_became_active":"2024-03-07T03:08:50.123620+0000","last_became_peered":"2024-03-07T03:08:50.123620+0000","last_unstale":"2024-03-07T03:09:03.388468+0000","last_undegraded":"2024-03-07T03:09:03.388468+0000","last_fullsized":"2024-03-07T03:09:03.388468+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:58:26.005440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.384355+0000","last_change":"2024-03-07T03:08:49.432285+0000","last_active":"2024-03-07T03:09:03.384355+0000","last_peered":"2024-03-07T03:09:03.384355+0000","last_clean":"2024-03-07T03:09:03.384355+0000","last_became_active":"2024-03-07T03:08:49.431326+0000","last_became_peered":"2024-03-07T03:08:49.431326+0000","last_unstale":"2024-03-07T03:09:03.384355+0000","last_undegraded":"2024-03-07T03:09:03.384355+0000","last_fullsized":"2024-03-07T03:09:03.384355+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:45:34.613738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.389378+0000","last_change":"2024-03-07T03:08:49.454753+0000","last_active":"2024-03-07T03:09:03.389378+0000","last_peered":"2024-03-07T03:09:03.389378+0000","last_clean":"2024-03-07T03:09:03.389378+0000","last_became_active":"2024-03-07T03:08:49.454237+0000","last_became_peered":"2024-03-07T03:08:49.454237+0000","last_unstale":"2024-03-07T03:09:03.389378+0000","last_undegraded":"2024-03-07T03:09:03.389378+0000","last_fullsized":"2024-03-07T03:09:03.389378+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:34:35.806144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.383974+0000","last_change":"2024-03-07T03:08:49.422878+0000","last_active":"2024-03-07T03:09:03.383974+0000","last_peered":"2024-03-07T03:09:03.383974+0000","last_clean":"2024-03-07T03:09:03.383974+0000","last_became_active":"2024-03-07T03:08:49.421940+0000","last_became_peered":"2024-03-07T03:08:49.421940+0000","last_unstale":"2024-03-07T03:09:03.383974+0000","last_undegraded":"2024-03-07T03:09:03.383974+0000","last_fullsized":"2024-03-07T03:09:03.383974+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:27:57.145332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.384952+0000","last_change":"2024-03-07T03:08:50.445879+0000","last_active":"2024-03-07T03:09:03.384952+0000","last_peered":"2024-03-07T03:09:03.384952+0000","last_clean":"2024-03-07T03:09:03.384952+0000","last_became_active":"2024-03-07T03:08:50.445045+0000","last_became_peered":"2024-03-07T03:08:50.445045+0000","last_unstale":"2024-03-07T03:09:03.384952+0000","last_undegraded":"2024-03-07T03:09:03.384952+0000","last_fullsized":"2024-03-07T03:09:03.384952+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:43:28.804667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.392269+0000","last_change":"2024-03-07T03:08:49.425773+0000","last_active":"2024-03-07T03:09:03.392269+0000","last_peered":"2024-03-07T03:09:03.392269+0000","last_clean":"2024-03-07T03:09:03.392269+0000","last_became_active":"2024-03-07T03:08:49.424985+0000","last_became_peered":"2024-03-07T03:08:49.424985+0000","last_unstale":"2024-03-07T03:09:03.392269+0000","last_undegraded":"2024-03-07T03:09:03.392269+0000","last_fullsized":"2024-03-07T03:09:03.392269+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:09:51.169893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388370+0000","last_change":"2024-03-07T03:08:49.451452+0000","last_active":"2024-03-07T03:09:03.388370+0000","last_peered":"2024-03-07T03:09:03.388370+0000","last_clean":"2024-03-07T03:09:03.388370+0000","last_became_active":"2024-03-07T03:08:49.450742+0000","last_became_peered":"2024-03-07T03:08:49.450742+0000","last_unstale":"2024-03-07T03:09:03.388370+0000","last_undegraded":"2024-03-07T03:09:03.388370+0000","last_fullsized":"2024-03-07T03:09:03.388370+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:33:02.521002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.390455+0000","last_change":"2024-03-07T03:08:49.414240+0000","last_active":"2024-03-07T03:09:03.390455+0000","last_peered":"2024-03-07T03:09:03.390455+0000","last_clean":"2024-03-07T03:09:03.390455+0000","last_became_active":"2024-03-07T03:08:49.413334+0000","last_became_peered":"2024-03-07T03:08:49.413334+0000","last_unstale":"2024-03-07T03:09:03.390455+0000","last_undegraded":"2024-03-07T03:09:03.390455+0000","last_fullsized":"2024-03-07T03:09:03.390455+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:04:00.717172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.979639+0000","last_change":"2024-03-07T03:06:42.667758+0000","last_active":"2024-03-07T03:09:03.979639+0000","last_peered":"2024-03-07T03:09:03.979639+0000","last_clean":"2024-03-07T03:09:03.979639+0000","last_became_active":"2024-03-07T03:06:42.667171+0000","last_became_peered":"2024-03-07T03:06:42.667171+0000","last_unstale":"2024-03-07T03:09:03.979639+0000","last_undegraded":"2024-03-07T03:09:03.979639+0000","last_fullsized":"2024-03-07T03:09:03.979639+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:55:07.432703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.376990+0000","last_change":"2024-03-07T03:08:50.476783+0000","last_active":"2024-03-07T03:09:03.376990+0000","last_peered":"2024-03-07T03:09:03.376990+0000","last_clean":"2024-03-07T03:09:03.376990+0000","last_became_active":"2024-03-07T03:08:50.476378+0000","last_became_peered":"2024-03-07T03:08:50.476378+0000","last_unstale":"2024-03-07T03:09:03.376990+0000","last_undegraded":"2024-03-07T03:09:03.376990+0000","last_fullsized":"2024-03-07T03:09:03.376990+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:50:41.163577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.387047+0000","last_change":"2024-03-07T03:06:42.794071+0000","last_active":"2024-03-07T03:09:03.387047+0000","last_peered":"2024-03-07T03:09:03.387047+0000","last_clean":"2024-03-07T03:09:03.387047+0000","last_became_active":"2024-03-07T03:06:42.792899+0000","last_became_peered":"2024-03-07T03:06:42.792899+0000","last_unstale":"2024-03-07T03:09:03.387047+0000","last_undegraded":"2024-03-07T03:09:03.387047+0000","last_fullsized":"2024-03-07T03:09:03.387047+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:07:21.918130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.378555+0000","last_change":"2024-03-07T03:08:49.437055+0000","last_active":"2024-03-07T03:09:03.378555+0000","last_peered":"2024-03-07T03:09:03.378555+0000","last_clean":"2024-03-07T03:09:03.378555+0000","last_became_active":"2024-03-07T03:08:49.436547+0000","last_became_peered":"2024-03-07T03:08:49.436547+0000","last_unstale":"2024-03-07T03:09:03.378555+0000","last_undegraded":"2024-03-07T03:09:03.378555+0000","last_fullsized":"2024-03-07T03:09:03.378555+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T15:03:30.849727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.392010+0000","last_change":"2024-03-07T03:08:50.478948+0000","last_active":"2024-03-07T03:09:03.392010+0000","last_peered":"2024-03-07T03:09:03.392010+0000","last_clean":"2024-03-07T03:09:03.392010+0000","last_became_active":"2024-03-07T03:08:50.478021+0000","last_became_peered":"2024-03-07T03:08:50.478021+0000","last_unstale":"2024-03-07T03:09:03.392010+0000","last_undegraded":"2024-03-07T03:09:03.392010+0000","last_fullsized":"2024-03-07T03:09:03.392010+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:14:54.695833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.387469+0000","last_change":"2024-03-07T03:08:49.425684+0000","last_active":"2024-03-07T03:09:03.387469+0000","last_peered":"2024-03-07T03:09:03.387469+0000","last_clean":"2024-03-07T03:09:03.387469+0000","last_became_active":"2024-03-07T03:08:49.424920+0000","last_became_peered":"2024-03-07T03:08:49.424920+0000","last_unstale":"2024-03-07T03:09:03.387469+0000","last_undegraded":"2024-03-07T03:09:03.387469+0000","last_fullsized":"2024-03-07T03:09:03.387469+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:24:18.065220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.379753+0000","last_change":"2024-03-07T03:06:42.792519+0000","last_active":"2024-03-07T03:09:03.379753+0000","last_peered":"2024-03-07T03:09:03.379753+0000","last_clean":"2024-03-07T03:09:03.379753+0000","last_became_active":"2024-03-07T03:06:42.792136+0000","last_became_peered":"2024-03-07T03:06:42.792136+0000","last_unstale":"2024-03-07T03:09:03.379753+0000","last_undegraded":"2024-03-07T03:09:03.379753+0000","last_fullsized":"2024-03-07T03:09:03.379753+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:50:03.686748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.390626+0000","last_change":"2024-03-07T03:08:50.443327+0000","last_active":"2024-03-07T03:09:03.390626+0000","last_peered":"2024-03-07T03:09:03.390626+0000","last_clean":"2024-03-07T03:09:03.390626+0000","last_became_active":"2024-03-07T03:08:50.442549+0000","last_became_peered":"2024-03-07T03:08:50.442549+0000","last_unstale":"2024-03-07T03:09:03.390626+0000","last_undegraded":"2024-03-07T03:09:03.390626+0000","last_fullsized":"2024-03-07T03:09:03.390626+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:31:57.940835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.389951+0000","last_change":"2024-03-07T03:08:49.424192+0000","last_active":"2024-03-07T03:09:03.389951+0000","last_peered":"2024-03-07T03:09:03.389951+0000","last_clean":"2024-03-07T03:09:03.389951+0000","last_became_active":"2024-03-07T03:08:49.421371+0000","last_became_peered":"2024-03-07T03:08:49.421371+0000","last_unstale":"2024-03-07T03:09:03.389951+0000","last_undegraded":"2024-03-07T03:09:03.389951+0000","last_fullsized":"2024-03-07T03:09:03.389951+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:21:00.758921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.379052+0000","last_change":"2024-03-07T03:06:42.668416+0000","last_active":"2024-03-07T03:09:03.379052+0000","last_peered":"2024-03-07T03:09:03.379052+0000","last_clean":"2024-03-07T03:09:03.379052+0000","last_became_active":"2024-03-07T03:06:42.668043+0000","last_became_peered":"2024-03-07T03:06:42.668043+0000","last_unstale":"2024-03-07T03:09:03.379052+0000","last_undegraded":"2024-03-07T03:09:03.379052+0000","last_fullsized":"2024-03-07T03:09:03.379052+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:07:59.168239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.979436+0000","last_change":"2024-03-07T03:08:50.484282+0000","last_active":"2024-03-07T03:09:03.979436+0000","last_peered":"2024-03-07T03:09:03.979436+0000","last_clean":"2024-03-07T03:09:03.979436+0000","last_became_active":"2024-03-07T03:08:50.483435+0000","last_became_peered":"2024-03-07T03:08:50.483435+0000","last_unstale":"2024-03-07T03:09:03.979436+0000","last_undegraded":"2024-03-07T03:09:03.979436+0000","last_fullsized":"2024-03-07T03:09:03.979436+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:43:13.608774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377381+0000","last_change":"2024-03-07T03:08:49.423282+0000","last_active":"2024-03-07T03:09:03.377381+0000","last_peered":"2024-03-07T03:09:03.377381+0000","last_clean":"2024-03-07T03:09:03.377381+0000","last_became_active":"2024-03-07T03:08:49.421827+0000","last_became_peered":"2024-03-07T03:08:49.421827+0000","last_unstale":"2024-03-07T03:09:03.377381+0000","last_undegraded":"2024-03-07T03:09:03.377381+0000","last_fullsized":"2024-03-07T03:09:03.377381+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T15:08:27.571976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":269,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:09:04.623234+0000","last_change":"2024-03-07T03:05:08.667040+0000","last_active":"2024-03-07T03:09:04.623234+0000","last_peered":"2024-03-07T03:09:04.623234+0000","last_clean":"2024-03-07T03:09:04.623234+0000","last_became_active":"2024-03-07T03:05:08.665926+0000","last_became_peered":"2024-03-07T03:05:08.665926+0000","last_unstale":"2024-03-07T03:09:04.623234+0000","last_undegraded":"2024-03-07T03:09:04.623234+0000","last_fullsized":"2024-03-07T03:09:04.623234+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:01:00.212577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:01:00.212577+0000","last_clean_scrub_stamp":"2024-03-07T03:01:00.212577+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:11:34.307802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.383577+0000","last_change":"2024-03-07T03:06:42.670236+0000","last_active":"2024-03-07T03:09:03.383577+0000","last_peered":"2024-03-07T03:09:03.383577+0000","last_clean":"2024-03-07T03:09:03.383577+0000","last_became_active":"2024-03-07T03:06:42.669808+0000","last_became_peered":"2024-03-07T03:06:42.669808+0000","last_unstale":"2024-03-07T03:09:03.383577+0000","last_undegraded":"2024-03-07T03:09:03.383577+0000","last_fullsized":"2024-03-07T03:09:03.383577+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:42:36.928386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.387442+0000","last_change":"2024-03-07T03:08:50.440449+0000","last_active":"2024-03-07T03:09:03.387442+0000","last_peered":"2024-03-07T03:09:03.387442+0000","last_clean":"2024-03-07T03:09:03.387442+0000","last_became_active":"2024-03-07T03:08:50.439241+0000","last_became_peered":"2024-03-07T03:08:50.439241+0000","last_unstale":"2024-03-07T03:09:03.387442+0000","last_undegraded":"2024-03-07T03:09:03.387442+0000","last_fullsized":"2024-03-07T03:09:03.387442+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:31:52.265284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.386338+0000","last_change":"2024-03-07T03:08:49.423091+0000","last_active":"2024-03-07T03:09:03.386338+0000","last_peered":"2024-03-07T03:09:03.386338+0000","last_clean":"2024-03-07T03:09:03.386338+0000","last_became_active":"2024-03-07T03:08:49.422236+0000","last_became_peered":"2024-03-07T03:08:49.422236+0000","last_unstale":"2024-03-07T03:09:03.386338+0000","last_undegraded":"2024-03-07T03:09:03.386338+0000","last_fullsized":"2024-03-07T03:09:03.386338+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:16:21.319520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.384189+0000","last_change":"2024-03-07T03:06:42.795963+0000","last_active":"2024-03-07T03:09:03.384189+0000","last_peered":"2024-03-07T03:09:03.384189+0000","last_clean":"2024-03-07T03:09:03.384189+0000","last_became_active":"2024-03-07T03:06:42.795033+0000","last_became_peered":"2024-03-07T03:06:42.795033+0000","last_unstale":"2024-03-07T03:09:03.384189+0000","last_undegraded":"2024-03-07T03:09:03.384189+0000","last_fullsized":"2024-03-07T03:09:03.384189+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:12:44.845051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:09:04.605356+0000","last_change":"2024-03-07T03:09:04.605356+0000","last_active":"2024-03-07T03:09:04.605356+0000","last_peered":"2024-03-07T03:09:04.605356+0000","last_clean":"2024-03-07T03:09:04.605356+0000","last_became_active":"2024-03-07T03:09:04.604227+0000","last_became_peered":"2024-03-07T03:09:04.604227+0000","last_unstale":"2024-03-07T03:09:04.605356+0000","last_undegraded":"2024-03-07T03:09:04.605356+0000","last_fullsized":"2024-03-07T03:09:04.605356+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:45:59.752727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388520+0000","last_change":"2024-03-07T03:08:49.433660+0000","last_active":"2024-03-07T03:09:03.388520+0000","last_peered":"2024-03-07T03:09:03.388520+0000","last_clean":"2024-03-07T03:09:03.388520+0000","last_became_active":"2024-03-07T03:08:49.433002+0000","last_became_peered":"2024-03-07T03:08:49.433002+0000","last_unstale":"2024-03-07T03:09:03.388520+0000","last_undegraded":"2024-03-07T03:09:03.388520+0000","last_fullsized":"2024-03-07T03:09:03.388520+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:07:11.391708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.383913+0000","last_change":"2024-03-07T03:06:42.667215+0000","last_active":"2024-03-07T03:09:03.383913+0000","last_peered":"2024-03-07T03:09:03.383913+0000","last_clean":"2024-03-07T03:09:03.383913+0000","last_became_active":"2024-03-07T03:06:42.666675+0000","last_became_peered":"2024-03-07T03:06:42.666675+0000","last_unstale":"2024-03-07T03:09:03.383913+0000","last_undegraded":"2024-03-07T03:09:03.383913+0000","last_fullsized":"2024-03-07T03:09:03.383913+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:35:31.916960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.386441+0000","last_change":"2024-03-07T03:08:50.478421+0000","last_active":"2024-03-07T03:09:03.386441+0000","last_peered":"2024-03-07T03:09:03.386441+0000","last_clean":"2024-03-07T03:09:03.386441+0000","last_became_active":"2024-03-07T03:08:50.477357+0000","last_became_peered":"2024-03-07T03:08:50.477357+0000","last_unstale":"2024-03-07T03:09:03.386441+0000","last_undegraded":"2024-03-07T03:09:03.386441+0000","last_fullsized":"2024-03-07T03:09:03.386441+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:23:13.285272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.384320+0000","last_change":"2024-03-07T03:08:49.427640+0000","last_active":"2024-03-07T03:09:03.384320+0000","last_peered":"2024-03-07T03:09:03.384320+0000","last_clean":"2024-03-07T03:09:03.384320+0000","last_became_active":"2024-03-07T03:08:49.426778+0000","last_became_peered":"2024-03-07T03:08:49.426778+0000","last_unstale":"2024-03-07T03:09:03.384320+0000","last_undegraded":"2024-03-07T03:09:03.384320+0000","last_fullsized":"2024-03-07T03:09:03.384320+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:44:19.233395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.387281+0000","last_change":"2024-03-07T03:06:42.666671+0000","last_active":"2024-03-07T03:09:03.387281+0000","last_peered":"2024-03-07T03:09:03.387281+0000","last_clean":"2024-03-07T03:09:03.387281+0000","last_became_active":"2024-03-07T03:06:42.666402+0000","last_became_peered":"2024-03-07T03:06:42.666402+0000","last_unstale":"2024-03-07T03:09:03.387281+0000","last_undegraded":"2024-03-07T03:09:03.387281+0000","last_fullsized":"2024-03-07T03:09:03.387281+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:28:33.638276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.387618+0000","last_change":"2024-03-07T03:08:50.482545+0000","last_active":"2024-03-07T03:09:03.387618+0000","last_peered":"2024-03-07T03:09:03.387618+0000","last_clean":"2024-03-07T03:09:03.387618+0000","last_became_active":"2024-03-07T03:08:50.481943+0000","last_became_peered":"2024-03-07T03:08:50.481943+0000","last_unstale":"2024-03-07T03:09:03.387618+0000","last_undegraded":"2024-03-07T03:09:03.387618+0000","last_fullsized":"2024-03-07T03:09:03.387618+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:46:06.527309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377109+0000","last_change":"2024-03-07T03:08:49.439651+0000","last_active":"2024-03-07T03:09:03.377109+0000","last_peered":"2024-03-07T03:09:03.377109+0000","last_clean":"2024-03-07T03:09:03.377109+0000","last_became_active":"2024-03-07T03:08:49.438883+0000","last_became_peered":"2024-03-07T03:08:49.438883+0000","last_unstale":"2024-03-07T03:09:03.377109+0000","last_undegraded":"2024-03-07T03:09:03.377109+0000","last_fullsized":"2024-03-07T03:09:03.377109+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:36:54.345992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.389084+0000","last_change":"2024-03-07T03:08:49.449533+0000","last_active":"2024-03-07T03:09:03.389084+0000","last_peered":"2024-03-07T03:09:03.389084+0000","last_clean":"2024-03-07T03:09:03.389084+0000","last_became_active":"2024-03-07T03:08:49.449003+0000","last_became_peered":"2024-03-07T03:08:49.449003+0000","last_unstale":"2024-03-07T03:09:03.389084+0000","last_undegraded":"2024-03-07T03:09:03.389084+0000","last_fullsized":"2024-03-07T03:09:03.389084+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:46:47.411367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388466+0000","last_change":"2024-03-07T03:08:49.424990+0000","last_active":"2024-03-07T03:09:03.388466+0000","last_peered":"2024-03-07T03:09:03.388466+0000","last_clean":"2024-03-07T03:09:03.388466+0000","last_became_active":"2024-03-07T03:08:49.423891+0000","last_became_peered":"2024-03-07T03:08:49.423891+0000","last_unstale":"2024-03-07T03:09:03.388466+0000","last_undegraded":"2024-03-07T03:09:03.388466+0000","last_fullsized":"2024-03-07T03:09:03.388466+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:06:49.623952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:09:04.605354+0000","last_change":"2024-03-07T03:09:04.605354+0000","last_active":"2024-03-07T03:09:04.605354+0000","last_peered":"2024-03-07T03:09:04.605354+0000","last_clean":"2024-03-07T03:09:04.605354+0000","last_became_active":"2024-03-07T03:09:04.604259+0000","last_became_peered":"2024-03-07T03:09:04.604259+0000","last_unstale":"2024-03-07T03:09:04.605354+0000","last_undegraded":"2024-03-07T03:09:04.605354+0000","last_fullsized":"2024-03-07T03:09:04.605354+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:15:28.053303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.979265+0000","last_change":"2024-03-07T03:08:49.440953+0000","last_active":"2024-03-07T03:09:03.979265+0000","last_peered":"2024-03-07T03:09:03.979265+0000","last_clean":"2024-03-07T03:09:03.979265+0000","last_became_active":"2024-03-07T03:08:49.440238+0000","last_became_peered":"2024-03-07T03:08:49.440238+0000","last_unstale":"2024-03-07T03:09:03.979265+0000","last_undegraded":"2024-03-07T03:09:03.979265+0000","last_fullsized":"2024-03-07T03:09:03.979265+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:56:52.219952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.389485+0000","last_change":"2024-03-07T03:08:49.454889+0000","last_active":"2024-03-07T03:09:03.389485+0000","last_peered":"2024-03-07T03:09:03.389485+0000","last_clean":"2024-03-07T03:09:03.389485+0000","last_became_active":"2024-03-07T03:08:49.454425+0000","last_became_peered":"2024-03-07T03:08:49.454425+0000","last_unstale":"2024-03-07T03:09:03.389485+0000","last_undegraded":"2024-03-07T03:09:03.389485+0000","last_fullsized":"2024-03-07T03:09:03.389485+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:00:02.972842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377619+0000","last_change":"2024-03-07T03:08:49.439517+0000","last_active":"2024-03-07T03:09:03.377619+0000","last_peered":"2024-03-07T03:09:03.377619+0000","last_clean":"2024-03-07T03:09:03.377619+0000","last_became_active":"2024-03-07T03:08:49.438707+0000","last_became_peered":"2024-03-07T03:08:49.438707+0000","last_unstale":"2024-03-07T03:09:03.377619+0000","last_undegraded":"2024-03-07T03:09:03.377619+0000","last_fullsized":"2024-03-07T03:09:03.377619+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:53:47.361431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:09:04.604412+0000","last_change":"2024-03-07T03:09:04.604412+0000","last_active":"2024-03-07T03:09:04.604412+0000","last_peered":"2024-03-07T03:09:04.604412+0000","last_clean":"2024-03-07T03:09:04.604412+0000","last_became_active":"2024-03-07T03:09:04.603338+0000","last_became_peered":"2024-03-07T03:09:04.603338+0000","last_unstale":"2024-03-07T03:09:04.604412+0000","last_undegraded":"2024-03-07T03:09:04.604412+0000","last_fullsized":"2024-03-07T03:09:04.604412+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:10:55.020596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.383871+0000","last_change":"2024-03-07T03:08:49.415515+0000","last_active":"2024-03-07T03:09:03.383871+0000","last_peered":"2024-03-07T03:09:03.383871+0000","last_clean":"2024-03-07T03:09:03.383871+0000","last_became_active":"2024-03-07T03:08:49.414747+0000","last_became_peered":"2024-03-07T03:08:49.414747+0000","last_unstale":"2024-03-07T03:09:03.383871+0000","last_undegraded":"2024-03-07T03:09:03.383871+0000","last_fullsized":"2024-03-07T03:09:03.383871+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:41:43.852936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.979123+0000","last_change":"2024-03-07T03:08:50.479999+0000","last_active":"2024-03-07T03:09:03.979123+0000","last_peered":"2024-03-07T03:09:03.979123+0000","last_clean":"2024-03-07T03:09:03.979123+0000","last_became_active":"2024-03-07T03:08:50.479113+0000","last_became_peered":"2024-03-07T03:08:50.479113+0000","last_unstale":"2024-03-07T03:09:03.979123+0000","last_undegraded":"2024-03-07T03:09:03.979123+0000","last_fullsized":"2024-03-07T03:09:03.979123+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:45:24.316898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.379942+0000","last_change":"2024-03-07T03:08:49.421752+0000","last_active":"2024-03-07T03:09:03.379942+0000","last_peered":"2024-03-07T03:09:03.379942+0000","last_clean":"2024-03-07T03:09:03.379942+0000","last_became_active":"2024-03-07T03:08:49.421165+0000","last_became_peered":"2024-03-07T03:08:49.421165+0000","last_unstale":"2024-03-07T03:09:03.379942+0000","last_undegraded":"2024-03-07T03:09:03.379942+0000","last_fullsized":"2024-03-07T03:09:03.379942+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:57:59.044174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388207+0000","last_change":"2024-03-07T03:08:49.454960+0000","last_active":"2024-03-07T03:09:03.388207+0000","last_peered":"2024-03-07T03:09:03.388207+0000","last_clean":"2024-03-07T03:09:03.388207+0000","last_became_active":"2024-03-07T03:08:49.454394+0000","last_became_peered":"2024-03-07T03:08:49.454394+0000","last_unstale":"2024-03-07T03:09:03.388207+0000","last_undegraded":"2024-03-07T03:09:03.388207+0000","last_fullsized":"2024-03-07T03:09:03.388207+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:39:23.039720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.380113+0000","last_change":"2024-03-07T03:08:49.437330+0000","last_active":"2024-03-07T03:09:03.380113+0000","last_peered":"2024-03-07T03:09:03.380113+0000","last_clean":"2024-03-07T03:09:03.380113+0000","last_became_active":"2024-03-07T03:08:49.436540+0000","last_became_peered":"2024-03-07T03:08:49.436540+0000","last_unstale":"2024-03-07T03:09:03.380113+0000","last_undegraded":"2024-03-07T03:09:03.380113+0000","last_fullsized":"2024-03-07T03:09:03.380113+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:45:32.346918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.383501+0000","last_change":"2024-03-07T03:08:50.110958+0000","last_active":"2024-03-07T03:09:03.383501+0000","last_peered":"2024-03-07T03:09:03.383501+0000","last_clean":"2024-03-07T03:09:03.383501+0000","last_became_active":"2024-03-07T03:08:50.110066+0000","last_became_peered":"2024-03-07T03:08:50.110066+0000","last_unstale":"2024-03-07T03:09:03.383501+0000","last_undegraded":"2024-03-07T03:09:03.383501+0000","last_fullsized":"2024-03-07T03:09:03.383501+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:57:28.837409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.380266+0000","last_change":"2024-03-07T03:08:49.428235+0000","last_active":"2024-03-07T03:09:03.380266+0000","last_peered":"2024-03-07T03:09:03.380266+0000","last_clean":"2024-03-07T03:09:03.380266+0000","last_became_active":"2024-03-07T03:08:49.427415+0000","last_became_peered":"2024-03-07T03:08:49.427415+0000","last_unstale":"2024-03-07T03:09:03.380266+0000","last_undegraded":"2024-03-07T03:09:03.380266+0000","last_fullsized":"2024-03-07T03:09:03.380266+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:22:23.578332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.378281+0000","last_change":"2024-03-07T03:08:50.479547+0000","last_active":"2024-03-07T03:09:03.378281+0000","last_peered":"2024-03-07T03:09:03.378281+0000","last_clean":"2024-03-07T03:09:03.378281+0000","last_became_active":"2024-03-07T03:08:50.478729+0000","last_became_peered":"2024-03-07T03:08:50.478729+0000","last_unstale":"2024-03-07T03:09:03.378281+0000","last_undegraded":"2024-03-07T03:09:03.378281+0000","last_fullsized":"2024-03-07T03:09:03.378281+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:20:56.825216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":32,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.979307+0000","last_change":"2024-03-07T03:08:49.418845+0000","last_active":"2024-03-07T03:09:03.979307+0000","last_peered":"2024-03-07T03:09:03.979307+0000","last_clean":"2024-03-07T03:09:03.979307+0000","last_became_active":"2024-03-07T03:08:49.416207+0000","last_became_peered":"2024-03-07T03:08:49.416207+0000","last_unstale":"2024-03-07T03:09:03.979307+0000","last_undegraded":"2024-03-07T03:09:03.979307+0000","last_fullsized":"2024-03-07T03:09:03.979307+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:50:57.585893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.390071+0000","last_change":"2024-03-07T03:08:50.486370+0000","last_active":"2024-03-07T03:09:03.390071+0000","last_peered":"2024-03-07T03:09:03.390071+0000","last_clean":"2024-03-07T03:09:03.390071+0000","last_became_active":"2024-03-07T03:08:50.486026+0000","last_became_peered":"2024-03-07T03:08:50.486026+0000","last_unstale":"2024-03-07T03:09:03.390071+0000","last_undegraded":"2024-03-07T03:09:03.390071+0000","last_fullsized":"2024-03-07T03:09:03.390071+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:22:43.621291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.378983+0000","last_change":"2024-03-07T03:08:49.427715+0000","last_active":"2024-03-07T03:09:03.378983+0000","last_peered":"2024-03-07T03:09:03.378983+0000","last_clean":"2024-03-07T03:09:03.378983+0000","last_became_active":"2024-03-07T03:08:49.426265+0000","last_became_peered":"2024-03-07T03:08:49.426265+0000","last_unstale":"2024-03-07T03:09:03.378983+0000","last_undegraded":"2024-03-07T03:09:03.378983+0000","last_fullsized":"2024-03-07T03:09:03.378983+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:19:02.652125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.384152+0000","last_change":"2024-03-07T03:08:50.111369+0000","last_active":"2024-03-07T03:09:03.384152+0000","last_peered":"2024-03-07T03:09:03.384152+0000","last_clean":"2024-03-07T03:09:03.384152+0000","last_became_active":"2024-03-07T03:08:50.110035+0000","last_became_peered":"2024-03-07T03:08:50.110035+0000","last_unstale":"2024-03-07T03:09:03.384152+0000","last_undegraded":"2024-03-07T03:09:03.384152+0000","last_fullsized":"2024-03-07T03:09:03.384152+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:39:27.353532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.389730+0000","last_change":"2024-03-07T03:08:49.428095+0000","last_active":"2024-03-07T03:09:03.389730+0000","last_peered":"2024-03-07T03:09:03.389730+0000","last_clean":"2024-03-07T03:09:03.389730+0000","last_became_active":"2024-03-07T03:08:49.426894+0000","last_became_peered":"2024-03-07T03:08:49.426894+0000","last_unstale":"2024-03-07T03:09:03.389730+0000","last_undegraded":"2024-03-07T03:09:03.389730+0000","last_fullsized":"2024-03-07T03:09:03.389730+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:35:45.666181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.379632+0000","last_change":"2024-03-07T03:08:50.479728+0000","last_active":"2024-03-07T03:09:03.379632+0000","last_peered":"2024-03-07T03:09:03.379632+0000","last_clean":"2024-03-07T03:09:03.379632+0000","last_became_active":"2024-03-07T03:08:50.477879+0000","last_became_peered":"2024-03-07T03:08:50.477879+0000","last_unstale":"2024-03-07T03:09:03.379632+0000","last_undegraded":"2024-03-07T03:09:03.379632+0000","last_fullsized":"2024-03-07T03:09:03.379632+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:54:21.740424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.379640+0000","last_change":"2024-03-07T03:08:49.427154+0000","last_active":"2024-03-07T03:09:03.379640+0000","last_peered":"2024-03-07T03:09:03.379640+0000","last_clean":"2024-03-07T03:09:03.379640+0000","last_became_active":"2024-03-07T03:08:49.426306+0000","last_became_peered":"2024-03-07T03:08:49.426306+0000","last_unstale":"2024-03-07T03:09:03.379640+0000","last_undegraded":"2024-03-07T03:09:03.379640+0000","last_fullsized":"2024-03-07T03:09:03.379640+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:21:17.304549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":8,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388748+0000","last_change":"2024-03-07T03:08:49.451933+0000","last_active":"2024-03-07T03:09:03.388748+0000","last_peered":"2024-03-07T03:09:03.388748+0000","last_clean":"2024-03-07T03:09:03.388748+0000","last_became_active":"2024-03-07T03:08:49.451466+0000","last_became_peered":"2024-03-07T03:08:49.451466+0000","last_unstale":"2024-03-07T03:09:03.388748+0000","last_undegraded":"2024-03-07T03:09:03.388748+0000","last_fullsized":"2024-03-07T03:09:03.388748+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:59:00.187710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"70'7","reported_seq":29,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.389525+0000","last_change":"2024-03-07T03:08:49.436124+0000","last_active":"2024-03-07T03:09:03.389525+0000","last_peered":"2024-03-07T03:09:03.389525+0000","last_clean":"2024-03-07T03:09:03.389525+0000","last_became_active":"2024-03-07T03:08:49.435086+0000","last_became_peered":"2024-03-07T03:08:49.435086+0000","last_unstale":"2024-03-07T03:09:03.389525+0000","last_undegraded":"2024-03-07T03:09:03.389525+0000","last_fullsized":"2024-03-07T03:09:03.389525+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:17:31.274262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.386340+0000","last_change":"2024-03-07T03:08:50.444679+0000","last_active":"2024-03-07T03:09:03.386340+0000","last_peered":"2024-03-07T03:09:03.386340+0000","last_clean":"2024-03-07T03:09:03.386340+0000","last_became_active":"2024-03-07T03:08:50.443875+0000","last_became_peered":"2024-03-07T03:08:50.443875+0000","last_unstale":"2024-03-07T03:09:03.386340+0000","last_undegraded":"2024-03-07T03:09:03.386340+0000","last_fullsized":"2024-03-07T03:09:03.386340+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:02:19.642787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":22,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.379845+0000","last_change":"2024-03-07T03:08:49.441798+0000","last_active":"2024-03-07T03:09:03.379845+0000","last_peered":"2024-03-07T03:09:03.379845+0000","last_clean":"2024-03-07T03:09:03.379845+0000","last_became_active":"2024-03-07T03:08:49.441261+0000","last_became_peered":"2024-03-07T03:08:49.441261+0000","last_unstale":"2024-03-07T03:09:03.379845+0000","last_undegraded":"2024-03-07T03:09:03.379845+0000","last_fullsized":"2024-03-07T03:09:03.379845+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:48:15.201462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377374+0000","last_change":"2024-03-07T03:08:50.478907+0000","last_active":"2024-03-07T03:09:03.377374+0000","last_peered":"2024-03-07T03:09:03.377374+0000","last_clean":"2024-03-07T03:09:03.377374+0000","last_became_active":"2024-03-07T03:08:50.478574+0000","last_became_peered":"2024-03-07T03:08:50.478574+0000","last_unstale":"2024-03-07T03:09:03.377374+0000","last_undegraded":"2024-03-07T03:09:03.377374+0000","last_fullsized":"2024-03-07T03:09:03.377374+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:23:54.091751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.383719+0000","last_change":"2024-03-07T03:08:49.407857+0000","last_active":"2024-03-07T03:09:03.383719+0000","last_peered":"2024-03-07T03:09:03.383719+0000","last_clean":"2024-03-07T03:09:03.383719+0000","last_became_active":"2024-03-07T03:08:49.407179+0000","last_became_peered":"2024-03-07T03:08:49.407179+0000","last_unstale":"2024-03-07T03:09:03.383719+0000","last_undegraded":"2024-03-07T03:09:03.383719+0000","last_fullsized":"2024-03-07T03:09:03.383719+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:54:49.068083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377124+0000","last_change":"2024-03-07T03:08:50.439815+0000","last_active":"2024-03-07T03:09:03.377124+0000","last_peered":"2024-03-07T03:09:03.377124+0000","last_clean":"2024-03-07T03:09:03.377124+0000","last_became_active":"2024-03-07T03:08:50.439269+0000","last_became_peered":"2024-03-07T03:08:50.439269+0000","last_unstale":"2024-03-07T03:09:03.377124+0000","last_undegraded":"2024-03-07T03:09:03.377124+0000","last_fullsized":"2024-03-07T03:09:03.377124+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:40:19.306830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.390874+0000","last_change":"2024-03-07T03:08:49.428343+0000","last_active":"2024-03-07T03:09:03.390874+0000","last_peered":"2024-03-07T03:09:03.390874+0000","last_clean":"2024-03-07T03:09:03.390874+0000","last_became_active":"2024-03-07T03:08:49.418275+0000","last_became_peered":"2024-03-07T03:08:49.418275+0000","last_unstale":"2024-03-07T03:09:03.390874+0000","last_undegraded":"2024-03-07T03:09:03.390874+0000","last_fullsized":"2024-03-07T03:09:03.390874+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:09:29.312397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.979524+0000","last_change":"2024-03-07T03:08:50.441016+0000","last_active":"2024-03-07T03:09:03.979524+0000","last_peered":"2024-03-07T03:09:03.979524+0000","last_clean":"2024-03-07T03:09:03.979524+0000","last_became_active":"2024-03-07T03:08:50.440186+0000","last_became_peered":"2024-03-07T03:08:50.440186+0000","last_unstale":"2024-03-07T03:09:03.979524+0000","last_undegraded":"2024-03-07T03:09:03.979524+0000","last_fullsized":"2024-03-07T03:09:03.979524+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:59:05.444181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377255+0000","last_change":"2024-03-07T03:08:49.433197+0000","last_active":"2024-03-07T03:09:03.377255+0000","last_peered":"2024-03-07T03:09:03.377255+0000","last_clean":"2024-03-07T03:09:03.377255+0000","last_became_active":"2024-03-07T03:08:49.426232+0000","last_became_peered":"2024-03-07T03:08:49.426232+0000","last_unstale":"2024-03-07T03:09:03.377255+0000","last_undegraded":"2024-03-07T03:09:03.377255+0000","last_fullsized":"2024-03-07T03:09:03.377255+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:48:13.717656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.390194+0000","last_change":"2024-03-07T03:08:50.486810+0000","last_active":"2024-03-07T03:09:03.390194+0000","last_peered":"2024-03-07T03:09:03.390194+0000","last_clean":"2024-03-07T03:09:03.390194+0000","last_became_active":"2024-03-07T03:08:50.486489+0000","last_became_peered":"2024-03-07T03:08:50.486489+0000","last_unstale":"2024-03-07T03:09:03.390194+0000","last_undegraded":"2024-03-07T03:09:03.390194+0000","last_fullsized":"2024-03-07T03:09:03.390194+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:11:50.953686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.386243+0000","last_change":"2024-03-07T03:08:49.442318+0000","last_active":"2024-03-07T03:09:03.386243+0000","last_peered":"2024-03-07T03:09:03.386243+0000","last_clean":"2024-03-07T03:09:03.386243+0000","last_became_active":"2024-03-07T03:08:49.441511+0000","last_became_peered":"2024-03-07T03:08:49.441511+0000","last_unstale":"2024-03-07T03:09:03.386243+0000","last_undegraded":"2024-03-07T03:09:03.386243+0000","last_fullsized":"2024-03-07T03:09:03.386243+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:21:40.063628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377802+0000","last_change":"2024-03-07T03:08:50.483512+0000","last_active":"2024-03-07T03:09:03.377802+0000","last_peered":"2024-03-07T03:09:03.377802+0000","last_clean":"2024-03-07T03:09:03.377802+0000","last_became_active":"2024-03-07T03:08:50.482850+0000","last_became_peered":"2024-03-07T03:08:50.482850+0000","last_unstale":"2024-03-07T03:09:03.377802+0000","last_undegraded":"2024-03-07T03:09:03.377802+0000","last_fullsized":"2024-03-07T03:09:03.377802+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:44:50.926405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388151+0000","last_change":"2024-03-07T03:08:49.435585+0000","last_active":"2024-03-07T03:09:03.388151+0000","last_peered":"2024-03-07T03:09:03.388151+0000","last_clean":"2024-03-07T03:09:03.388151+0000","last_became_active":"2024-03-07T03:08:49.433630+0000","last_became_peered":"2024-03-07T03:08:49.433630+0000","last_unstale":"2024-03-07T03:09:03.388151+0000","last_undegraded":"2024-03-07T03:09:03.388151+0000","last_fullsized":"2024-03-07T03:09:03.388151+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:56:19.003898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":77,"num_read_kb":80,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201349,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28352,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716800,"statfs":{"total":95995035648,"available":95966003200,"internally_reserved":0,"allocated":1359872,"data_stored":916402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.92600000000000005,"15min":0.92100000000000004},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.2410000000000001,"5min":1.524,"15min":1.524},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.95599999999999996,"15min":1.016},"min":{"1min":0.59599999999999997,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.2290000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":1.2929999999999999}]},{"osd":1,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.89200000000000002,"15min":0.92000000000000004},"min":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.0840000000000001,"5min":1.28,"15min":1.28},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.014,"15min":1.0509999999999999},"min":{"1min":0.64600000000000002,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.3280000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":1.147}]},{"osd":2,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.0049999999999999,"15min":1.044},"min":{"1min":0.57699999999999996,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.3899999999999999,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":1.052},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.99399999999999999,"15min":0.98799999999999999},"min":{"1min":0.72699999999999998,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.329,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":1.079}]},{"osd":3,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.91900000000000004,"15min":0.94099999999999995},"min":{"1min":0.59999999999999998,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.169,"5min":1.327,"15min":1.327},"last":1.024},{"interface":"front","average":{"1min":1.038,"5min":1.0549999999999999,"15min":1.079},"min":{"1min":0.84299999999999997,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.194,"5min":1.448,"15min":1.448},"last":0.99299999999999999}]},{"osd":4,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.95799999999999996,"15min":0.93000000000000005},"min":{"1min":0.71899999999999997,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.4810000000000001,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.999,"5min":0.99199999999999999,"15min":0.99199999999999999},"min":{"1min":0.68899999999999995,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.2549999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.117}]},{"osd":5,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.0209999999999999,"15min":0.996},"min":{"1min":0.80600000000000005,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.3879999999999999,"5min":1.446,"15min":1.446},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.079,"5min":1.0900000000000001,"15min":1.113},"min":{"1min":0.77400000000000002,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.4359999999999999,"5min":1.554,"15min":1.554},"last":1.2070000000000001}]},{"osd":6,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":1.135,"15min":1.139},"min":{"1min":0.88500000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.522,"5min":1.522,"15min":1.522},"last":1.2509999999999999},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.135,"15min":1.1240000000000001},"min":{"1min":0.77300000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.4419999999999999,"5min":1.496,"15min":1.496},"last":1.1850000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462991,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28336,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716816,"statfs":{"total":95995035648,"available":95966019584,"internally_reserved":0,"allocated":1343488,"data_stored":915983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.874,"15min":0.91600000000000004},"min":{"1min":0.55000000000000004,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.175,"5min":1.349,"15min":1.349},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.91600000000000004,"15min":0.91500000000000004},"min":{"1min":0.64600000000000002,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.29,"5min":1.496,"15min":1.496},"last":0.99099999999999999}]},{"osd":1,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.94599999999999995,"15min":0.91800000000000004},"min":{"1min":0.58299999999999996,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.2729999999999999,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.0880000000000001},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.93700000000000006,"15min":0.93400000000000005},"min":{"1min":0.64900000000000002,"5min":0.438,"15min":0.438},"max":{"1min":1.2130000000000001,"5min":1.454,"15min":1.4570000000000001},"last":1.044}]},{"osd":2,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.94399999999999995,"15min":0.91400000000000003},"min":{"1min":0.68100000000000005,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.0089999999999999,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.042,"5min":1.006,"15min":0.97399999999999998},"min":{"1min":0.88700000000000001,"5min":0.59399999999999997,"15min":0.55600000000000005},"max":{"1min":1.1699999999999999,"5min":3.6360000000000001,"15min":3.6360000000000001},"last":0.94399999999999995}]},{"osd":3,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.0009999999999999,"15min":0.96099999999999997},"min":{"1min":0.68500000000000005,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.1950000000000001,"5min":3.9169999999999998,"15min":3.9169999999999998},"last":1.1950000000000001},{"interface":"front","average":{"1min":1.038,"5min":1.042,"15min":1.012},"min":{"1min":0.83299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.264,"5min":3.4790000000000001,"15min":3.4790000000000001},"last":0.97099999999999997}]},{"osd":4,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.94099999999999995,"15min":0.84599999999999997},"min":{"1min":0.68600000000000005,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.387,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.68600000000000005},{"interface":"front","average":{"1min":1.099,"5min":1.008,"15min":0.92200000000000004},"min":{"1min":0.86799999999999999,"5min":0.307,"15min":0.307},"max":{"1min":1.409,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.0049999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.0369999999999999,"15min":0.93500000000000005},"min":{"1min":0.80000000000000004,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.248,"5min":2.1720000000000002,"15min":2.1720000000000002},"last":0.84199999999999997},{"interface":"front","average":{"1min":1.087,"5min":1.0529999999999999,"15min":0.98899999999999999},"min":{"1min":0.78600000000000003,"5min":0.374,"15min":0.374},"max":{"1min":1.3859999999999999,"5min":1.587,"15min":1.587},"last":0.80400000000000005}]},{"osd":7,"last update":"Thu Mar 7 03:10:16 2024","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.099,"15min":1.0409999999999999},"min":{"1min":0.96799999999999997,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.4790000000000001,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.1399999999999999},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.1479999999999999,"15min":1.1000000000000001},"min":{"1min":0.74399999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.369,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":1.1679999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574977,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27748,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717404,"statfs":{"total":95995035648,"available":95966621696,"internally_reserved":0,"allocated":741376,"data_stored":324585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.68999999999999995,"15min":0.61799999999999999},"min":{"1min":0.38300000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.95099999999999996,"5min":1.4099999999999999,"15min":1.4099999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.77800000000000002,"15min":0.65800000000000003},"min":{"1min":0.372,"5min":0.34699999999999998,"15min":0.28899999999999998},"max":{"1min":1.244,"5min":1.244,"15min":1.2569999999999999},"last":1.0900000000000001}]},{"osd":2,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.76800000000000002,"15min":0.69399999999999995},"min":{"1min":0.42999999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.1970000000000001,"5min":1.1970000000000001,"15min":1.3169999999999999},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.78800000000000003,"5min":0.78600000000000003,"15min":0.68300000000000005},"min":{"1min":0.51700000000000002,"5min":0.308,"15min":0.308},"max":{"1min":1.097,"5min":1.327,"15min":1.327},"last":0.80700000000000005}]},{"osd":3,"last update":"Thu Mar 7 03:10:13 2024","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.84699999999999998,"15min":0.82099999999999995},"min":{"1min":0.65000000000000002,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.2529999999999999,"5min":1.403,"15min":1.403},"last":1.131},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.79100000000000004,"15min":0.78600000000000003},"min":{"1min":0.63400000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.103,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.71199999999999997}]},{"osd":4,"last update":"Thu Mar 7 03:09:57 2024","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":0.97499999999999998,"15min":0.94699999999999995},"min":{"1min":0.79100000000000004,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.2929999999999999,"5min":1.53,"15min":1.53},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0109999999999999,"15min":0.94199999999999995},"min":{"1min":0.76900000000000002,"5min":0.503,"15min":0.503},"max":{"1min":1.3100000000000001,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":1.2789999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:09:37 2024","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1,"15min":0.96599999999999997},"min":{"1min":0.78100000000000003,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.498,"5min":1.498,"15min":1.498},"last":1.0309999999999999},{"interface":"front","average":{"1min":1.079,"5min":1.0649999999999999,"15min":1.046},"min":{"1min":0.89800000000000002,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.377,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.2470000000000001}]},{"osd":6,"last update":"Thu Mar 7 03:10:25 2024","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0720000000000001,"15min":1.0840000000000001},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.57799999999999996},"max":{"1min":1.393,"5min":1.5109999999999999,"15min":1.8520000000000001},"last":1.357},{"interface":"front","average":{"1min":1.069,"5min":1.0760000000000001,"15min":1.1040000000000001},"min":{"1min":0.72699999999999998,"5min":0.65800000000000003,"15min":0.60899999999999999},"max":{"1min":1.2629999999999999,"5min":1.4990000000000001,"15min":1.512},"last":1.175}]},{"osd":7,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.137,"15min":1.175},"min":{"1min":0.75600000000000001,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.4770000000000001,"5min":1.619,"15min":1.619},"last":1.3169999999999999},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.1479999999999999,"15min":1.1399999999999999},"min":{"1min":0.94199999999999995,"5min":0.622,"15min":0.622},"max":{"1min":1.508,"5min":1.508,"15min":1.508},"last":1.4019999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738507,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28332,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716820,"statfs":{"total":95995035648,"available":95966023680,"internally_reserved":0,"allocated":1343488,"data_stored":915557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Mar 7 03:10:33 2024","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.71699999999999997,"15min":0.623},"min":{"1min":0.42899999999999999,"5min":0.34300000000000003,"15min":0.25},"max":{"1min":1.073,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.75,"15min":0.67200000000000004},"min":{"1min":0.442,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.1339999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.872}]},{"osd":2,"last update":"Thu Mar 7 03:10:05 2024","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.79100000000000004,"15min":0.70399999999999996},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.28499999999999998},"max":{"1min":1.2190000000000001,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.79000000000000004,"5min":0.80500000000000005,"15min":0.73699999999999999},"min":{"1min":0.35999999999999999,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.4279999999999999},"last":0.747}]},{"osd":3,"last update":"Thu Mar 7 03:10:05 2024","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.86799999999999999,"15min":0.82099999999999995},"min":{"1min":0.49299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.1559999999999999,"5min":1.276,"15min":1.6180000000000001},"last":0.84999999999999998},{"interface":"front","average":{"1min":0.872,"5min":0.89000000000000001,"15min":0.83699999999999997},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.316},"max":{"1min":1.175,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":0.81299999999999994}]},{"osd":4,"last update":"Thu Mar 7 03:09:55 2024","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0089999999999999,"15min":0.96699999999999997},"min":{"1min":0.75800000000000001,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.506,"5min":1.506,"15min":1.506},"last":0.97299999999999998},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.92900000000000005,"15min":0.91400000000000003},"min":{"1min":0.66200000000000003,"5min":0.498,"15min":0.498},"max":{"1min":1.169,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":1.0580000000000001}]},{"osd":5,"last update":"Thu Mar 7 03:09:38 2024","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.0920000000000001,"15min":1.0089999999999999},"min":{"1min":0.74199999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.4750000000000001,"5min":1.8109999999999999,"15min":1.8109999999999999},"last":0.94999999999999996},{"interface":"front","average":{"1min":1.161,"5min":1.079,"15min":1.0149999999999999},"min":{"1min":0.64800000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.712,"5min":1.712,"15min":1.712},"last":1.028}]},{"osd":6,"last update":"Thu Mar 7 03:10:23 2024","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.02,"15min":1.022},"min":{"1min":0.68100000000000005,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.3080000000000001,"5min":1.6779999999999999,"15min":1.6779999999999999},"last":0.91600000000000004},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.1679999999999999,"15min":1.1910000000000001},"min":{"1min":0.92700000000000005,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.466,"5min":1.8340000000000001,"15min":1.8340000000000001},"last":1.087}]},{"osd":7,"last update":"Thu Mar 7 03:10:20 2024","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.0629999999999999,"15min":1.0649999999999999},"min":{"1min":0.73299999999999998,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.3999999999999999,"5min":1.518,"15min":1.518},"last":0.99399999999999999},{"interface":"front","average":{"1min":1.179,"5min":1.181,"15min":1.194},"min":{"1min":0.77800000000000002,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.4350000000000001,"5min":1.752,"15min":1.752},"last":1.143}]}]},{"osd":2,"up_from":18,"seq":77309411447,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27752,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717400,"statfs":{"total":95995035648,"available":95966617600,"internally_reserved":0,"allocated":745472,"data_stored":324928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:13 2024","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.70799999999999996,"15min":0.63800000000000001},"min":{"1min":0.33800000000000002,"5min":0.318,"15min":0.307},"max":{"1min":1.169,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.76900000000000002,"15min":0.68500000000000005},"min":{"1min":0.375,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.0640000000000001,"5min":1.431,"15min":1.431},"last":0.83799999999999997}]},{"osd":1,"last update":"Thu Mar 7 03:10:13 2024","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.71799999999999997,"15min":0.68400000000000005},"min":{"1min":0.47199999999999998,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":0.90200000000000002,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.79200000000000004,"15min":0.72499999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.28999999999999998},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.90900000000000003}]},{"osd":3,"last update":"Thu Mar 7 03:10:13 2024","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.83199999999999996,"15min":0.81699999999999995},"min":{"1min":0.443,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.089,"5min":1.222,"15min":1.2509999999999999},"last":0.997},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.84899999999999998,"15min":0.81799999999999995},"min":{"1min":0.41199999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.2110000000000001,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":0.72299999999999998}]},{"osd":4,"last update":"Thu Mar 7 03:09:50 2024","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.0049999999999999,"15min":0.96499999999999997},"min":{"1min":0.72799999999999998,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.256,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":1.087},{"interface":"front","average":{"1min":1.024,"5min":0.995,"15min":0.93799999999999994},"min":{"1min":0.746,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.3799999999999999,"5min":1.649,"15min":1.649},"last":1.1060000000000001}]},{"osd":5,"last update":"Thu Mar 7 03:09:42 2024","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.034,"15min":0.98799999999999999},"min":{"1min":0.84799999999999998,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.3460000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.0149999999999999},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.0309999999999999,"15min":0.999},"min":{"1min":0.71499999999999997,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.4670000000000001,"5min":1.53,"15min":1.53},"last":1.1579999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:10:27 2024","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.052,"15min":1.0740000000000001},"min":{"1min":0.64900000000000002,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.341,"5min":1.599,"15min":1.599},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.161,"5min":1.139,"15min":1.127},"min":{"1min":0.77200000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.5589999999999999,"5min":1.746,"15min":1.746},"last":0.96299999999999997}]},{"osd":7,"last update":"Thu Mar 7 03:10:13 2024","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.111,"15min":1.099},"min":{"1min":0.63700000000000001,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.3720000000000001,"5min":1.6759999999999999,"15min":1.6759999999999999},"last":1.1779999999999999},{"interface":"front","average":{"1min":1.133,"5min":1.1639999999999999,"15min":1.1579999999999999},"min":{"1min":0.64000000000000001,"5min":0.501,"15min":0.501},"max":{"1min":1.423,"5min":1.696,"15min":1.696},"last":1.056}]}]},{"osd":3,"up_from":26,"seq":111669149803,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27748,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717404,"statfs":{"total":95995035648,"available":95966621696,"internally_reserved":0,"allocated":741376,"data_stored":324402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.67700000000000005,"15min":0.68400000000000005},"min":{"1min":0.33400000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.0800000000000001,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":1.0740000000000001},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.74299999999999999,"15min":0.748},"min":{"1min":0.38700000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.7110000000000001,"5min":1.7110000000000001,"15min":1.7110000000000001},"last":0.86499999999999999}]},{"osd":1,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.77100000000000002,"15min":0.78500000000000003},"min":{"1min":0.439,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.1559999999999999,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.76200000000000001,"15min":0.76000000000000001},"min":{"1min":0.42699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.1339999999999999,"5min":1.476,"15min":1.476},"last":0.65500000000000003}]},{"osd":2,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.81899999999999995,"15min":0.82099999999999995},"min":{"1min":0.371,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.1559999999999999,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.86599999999999999,"5min":0.79500000000000004,"15min":0.78000000000000003},"min":{"1min":0.40999999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.5269999999999999,"5min":1.5269999999999999,"15min":1.5269999999999999},"last":1.111}]},{"osd":4,"last update":"Thu Mar 7 03:09:55 2024","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.97199999999999998,"15min":0.91200000000000003},"min":{"1min":0.75700000000000001,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.452,"5min":1.452,"15min":1.452},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.0209999999999999,"5min":0.99399999999999999,"15min":0.95699999999999996},"min":{"1min":0.84999999999999998,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.2729999999999999,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":1.2689999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:09:42 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":0.97899999999999998,"15min":0.97099999999999997},"min":{"1min":0.66300000000000003,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":1.1859999999999999},{"interface":"front","average":{"1min":1.012,"5min":1.03,"15min":1.0740000000000001},"min":{"1min":0.85599999999999998,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.2889999999999999,"5min":1.4590000000000001,"15min":1.492},"last":1.1439999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:10:31 2024","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.054,"15min":1.0700000000000001},"min":{"1min":0.98099999999999998,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.4419999999999999,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":1.3089999999999999},{"interface":"front","average":{"1min":1.099,"5min":1.0700000000000001,"15min":1.0660000000000001},"min":{"1min":0.90200000000000002,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.3129999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":0.90200000000000002}]},{"osd":7,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.0629999999999999,"15min":1.097},"min":{"1min":0.63100000000000001,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.4139999999999999,"5min":1.45,"15min":1.45},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.113,"15min":1.125},"min":{"1min":0.69899999999999995,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.3260000000000001,"5min":1.512,"15min":1.512},"last":1.2190000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986274,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":761856,"data_stored":326022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:09:47 2024","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.84399999999999997,"15min":0.85599999999999998},"min":{"1min":0.51800000000000002,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.105,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.94999999999999996},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.90500000000000003,"15min":0.86799999999999999},"min":{"1min":0.68100000000000005,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.29,"5min":1.2909999999999999,"15min":1.323},"last":1.1879999999999999}]},{"osd":1,"last update":"Thu Mar 7 03:09:47 2024","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.86899999999999999,"15min":0.86899999999999999},"min":{"1min":0.626,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.135,"5min":1.194,"15min":1.194},"last":1.1559999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.96299999999999997,"15min":0.91000000000000003},"min":{"1min":0.72599999999999998,"5min":0.52700000000000002,"15min":0.498},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":0.91000000000000003}]},{"osd":2,"last update":"Thu Mar 7 03:09:47 2024","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.006,"15min":0.92700000000000005},"min":{"1min":0.78300000000000003,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.3540000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.91500000000000004,"15min":0.87},"min":{"1min":0.77600000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.302,"5min":1.302,"15min":1.3720000000000001},"last":1.1160000000000001}]},{"osd":3,"last update":"Thu Mar 7 03:09:47 2024","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":0.94699999999999995,"15min":0.90800000000000003},"min":{"1min":0.77000000000000002,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.2569999999999999,"5min":1.2849999999999999,"15min":1.286},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.878,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.61699999999999999,"5min":0.504,"15min":0.504},"max":{"1min":1.1040000000000001,"5min":1.27,"15min":1.27},"last":0.72799999999999998}]},{"osd":5,"last update":"Thu Mar 7 03:09:35 2024","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":0.90000000000000002,"15min":0.76400000000000001},"min":{"1min":0.72399999999999998,"5min":0.374,"15min":0.33700000000000002},"max":{"1min":1.29,"5min":1.349,"15min":1.349},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.90000000000000002,"15min":0.79000000000000004},"min":{"1min":0.70899999999999996,"5min":0.41099999999999998,"15min":0.374},"max":{"1min":1.385,"5min":1.385,"15min":1.385},"last":1.002}]},{"osd":6,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":0.97299999999999998,"15min":0.89000000000000001},"min":{"1min":0.89600000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.2909999999999999,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":1.2130000000000001},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.069,"15min":0.97899999999999998},"min":{"1min":0.73399999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.492,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.0349999999999999}]},{"osd":7,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.089,"15min":1.0620000000000001},"min":{"1min":0.94699999999999995,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.357,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":1.0880000000000001},{"interface":"front","average":{"1min":1.052,"5min":1.0009999999999999,"15min":0.97899999999999998},"min":{"1min":0.85899999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.264,"5min":1.419,"15min":1.419},"last":1.0589999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724632,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27752,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717400,"statfs":{"total":95995035648,"available":95966617600,"internally_reserved":0,"allocated":745472,"data_stored":324906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:09:37 2024","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.89000000000000001,"15min":0.83999999999999997},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.304,"5min":1.5189999999999999,"15min":1.665},"last":1.2},{"interface":"front","average":{"1min":0.91000000000000003,"5min":0.90900000000000003,"15min":0.86199999999999999},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.22,"5min":2.359,"15min":2.359},"last":1.1399999999999999}]},{"osd":1,"last update":"Thu Mar 7 03:09:37 2024","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.93799999999999994,"15min":0.876},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.143,"5min":1.276,"15min":1.276},"last":1.0740000000000001},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.90500000000000003,"15min":0.88100000000000001},"min":{"1min":0.55900000000000005,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.26,"5min":2.0169999999999999,"15min":2.0169999999999999},"last":1.244}]},{"osd":2,"last update":"Thu Mar 7 03:09:37 2024","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.93500000000000005,"15min":0.93600000000000005},"min":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.2869999999999999,"5min":1.9610000000000001,"15min":1.9610000000000001},"last":0.99399999999999999},{"interface":"front","average":{"1min":1.046,"5min":0.98099999999999998,"15min":0.90300000000000002},"min":{"1min":0.57199999999999995,"5min":0.438,"15min":0.438},"max":{"1min":1.393,"5min":1.9770000000000001,"15min":1.9770000000000001},"last":0.95799999999999996}]},{"osd":3,"last update":"Thu Mar 7 03:09:37 2024","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.97299999999999998,"15min":0.93200000000000005},"min":{"1min":0.58299999999999996,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.29,"5min":1.419,"15min":1.419},"last":1.171},{"interface":"front","average":{"1min":1.016,"5min":1.004,"15min":0.95699999999999996},"min":{"1min":0.59799999999999998,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.341,"5min":2.0720000000000001,"15min":2.0720000000000001},"last":1.29}]},{"osd":4,"last update":"Thu Mar 7 03:09:37 2024","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.83999999999999997,"15min":0.81599999999999995},"min":{"1min":0.373,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.0640000000000001,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.69799999999999995},{"interface":"front","average":{"1min":1.0129999999999999,"5min":0.96999999999999997,"15min":0.874},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.4339999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":1.107}]},{"osd":6,"last update":"Thu Mar 7 03:10:30 2024","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.0069999999999999,"15min":0.98799999999999999},"min":{"1min":0.84699999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.351,"5min":1.546,"15min":1.546},"last":1.272},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.0229999999999999,"15min":0.995},"min":{"1min":0.66600000000000004,"5min":0.379,"15min":0.379},"max":{"1min":1.401,"5min":1.627,"15min":1.627},"last":0.85299999999999998}]},{"osd":7,"last update":"Thu Mar 7 03:10:12 2024","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0189999999999999,"15min":0.96499999999999997},"min":{"1min":0.73999999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.355,"5min":1.51,"15min":1.51},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.157,"5min":1.1120000000000001,"15min":1.0349999999999999},"min":{"1min":0.89400000000000002,"5min":0.435,"15min":0.435},"max":{"1min":1.373,"5min":1.782,"15min":1.782},"last":1.3220000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-07T03:10:38.163 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph pg dump --format=json 2024-03-07T03:10:40.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:40 smithi145 bash[21339]: cluster 2024-03-07T03:10:38.444573+0000 mgr.y (mgr.24302) 179 : cluster 0 pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:40.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:40 smithi145 bash[30906]: cluster 2024-03-07T03:10:38.444573+0000 mgr.y (mgr.24302) 179 : cluster 0 pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:40.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:40 smithi189 bash[23614]: cluster 2024-03-07T03:10:38.444573+0000 mgr.y (mgr.24302) 179 : cluster 0 pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:42.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:42 smithi145 bash[21339]: cluster 2024-03-07T03:10:40.445671+0000 mgr.y (mgr.24302) 180 : cluster 0 pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:42.813 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:10:42 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:10:42] "GET /metrics HTTP/1.1" 200 33768 "" "Prometheus/2.43.0" 2024-03-07T03:10:42.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:42 smithi145 bash[30906]: cluster 2024-03-07T03:10:40.445671+0000 mgr.y (mgr.24302) 180 : cluster 0 pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:42.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:42 smithi189 bash[23614]: cluster 2024-03-07T03:10:40.445671+0000 mgr.y (mgr.24302) 180 : cluster 0 pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:42.894 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:44.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:44 smithi145 bash[21339]: cluster 2024-03-07T03:10:42.446288+0000 mgr.y (mgr.24302) 181 : cluster 0 pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:44.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:44 smithi145 bash[30906]: cluster 2024-03-07T03:10:42.446288+0000 mgr.y (mgr.24302) 181 : cluster 0 pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:44.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:44 smithi189 bash[23614]: cluster 2024-03-07T03:10:42.446288+0000 mgr.y (mgr.24302) 181 : cluster 0 pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:46.283 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:10:46.285 INFO:teuthology.orchestra.run.smithi145.stderr:dumped all 2024-03-07T03:10:46.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:46 smithi145 bash[21339]: cluster 2024-03-07T03:10:44.447339+0000 mgr.y (mgr.24302) 182 : cluster 0 pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:46.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:46 smithi145 bash[30906]: cluster 2024-03-07T03:10:44.447339+0000 mgr.y (mgr.24302) 182 : cluster 0 pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:46.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:46 smithi189 bash[23614]: cluster 2024-03-07T03:10:44.447339+0000 mgr.y (mgr.24302) 182 : cluster 0 pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:47.085 INFO:teuthology.orchestra.run.smithi145.stdout:{"pg_ready":true,"pg_map":{"version":156,"stamp":"2024-03-07T03:10:44.446541+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592730,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":289,"num_read_kb":386,"num_write":224,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173,"ondisk_log_size":173,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":223788,"kb_used_data":7600,"kb_used_omap":13,"kb_used_meta":216050,"kb_avail":749737428,"statfs":{"total":767960285184,"available":767731126272,"internally_reserved":0,"allocated":7782400,"data_stored":4372785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14248,"internal_metadata":221235288},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004278"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.387813+0000","last_change":"2024-03-07T03:08:49.451405+0000","last_active":"2024-03-07T03:09:03.387813+0000","last_peered":"2024-03-07T03:09:03.387813+0000","last_clean":"2024-03-07T03:09:03.387813+0000","last_became_active":"2024-03-07T03:08:49.450204+0000","last_became_peered":"2024-03-07T03:08:49.450204+0000","last_unstale":"2024-03-07T03:09:03.387813+0000","last_undegraded":"2024-03-07T03:09:03.387813+0000","last_fullsized":"2024-03-07T03:09:03.387813+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:20:33.165811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.390799+0000","last_change":"2024-03-07T03:08:49.428857+0000","last_active":"2024-03-07T03:09:03.390799+0000","last_peered":"2024-03-07T03:09:03.390799+0000","last_clean":"2024-03-07T03:09:03.390799+0000","last_became_active":"2024-03-07T03:08:49.426339+0000","last_became_peered":"2024-03-07T03:08:49.426339+0000","last_unstale":"2024-03-07T03:09:03.390799+0000","last_undegraded":"2024-03-07T03:09:03.390799+0000","last_fullsized":"2024-03-07T03:09:03.390799+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:04:00.929586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377229+0000","last_change":"2024-03-07T03:08:50.443295+0000","last_active":"2024-03-07T03:09:03.377229+0000","last_peered":"2024-03-07T03:09:03.377229+0000","last_clean":"2024-03-07T03:09:03.377229+0000","last_became_active":"2024-03-07T03:08:50.442492+0000","last_became_peered":"2024-03-07T03:08:50.442492+0000","last_unstale":"2024-03-07T03:09:03.377229+0000","last_undegraded":"2024-03-07T03:09:03.377229+0000","last_fullsized":"2024-03-07T03:09:03.377229+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:42:15.472843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377993+0000","last_change":"2024-03-07T03:08:49.420961+0000","last_active":"2024-03-07T03:09:03.377993+0000","last_peered":"2024-03-07T03:09:03.377993+0000","last_clean":"2024-03-07T03:09:03.377993+0000","last_became_active":"2024-03-07T03:08:49.420313+0000","last_became_peered":"2024-03-07T03:08:49.420313+0000","last_unstale":"2024-03-07T03:09:03.377993+0000","last_undegraded":"2024-03-07T03:09:03.377993+0000","last_fullsized":"2024-03-07T03:09:03.377993+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:13:19.978685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388468+0000","last_change":"2024-03-07T03:08:50.124290+0000","last_active":"2024-03-07T03:09:03.388468+0000","last_peered":"2024-03-07T03:09:03.388468+0000","last_clean":"2024-03-07T03:09:03.388468+0000","last_became_active":"2024-03-07T03:08:50.123620+0000","last_became_peered":"2024-03-07T03:08:50.123620+0000","last_unstale":"2024-03-07T03:09:03.388468+0000","last_undegraded":"2024-03-07T03:09:03.388468+0000","last_fullsized":"2024-03-07T03:09:03.388468+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:58:26.005440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.384355+0000","last_change":"2024-03-07T03:08:49.432285+0000","last_active":"2024-03-07T03:09:03.384355+0000","last_peered":"2024-03-07T03:09:03.384355+0000","last_clean":"2024-03-07T03:09:03.384355+0000","last_became_active":"2024-03-07T03:08:49.431326+0000","last_became_peered":"2024-03-07T03:08:49.431326+0000","last_unstale":"2024-03-07T03:09:03.384355+0000","last_undegraded":"2024-03-07T03:09:03.384355+0000","last_fullsized":"2024-03-07T03:09:03.384355+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:45:34.613738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.389378+0000","last_change":"2024-03-07T03:08:49.454753+0000","last_active":"2024-03-07T03:09:03.389378+0000","last_peered":"2024-03-07T03:09:03.389378+0000","last_clean":"2024-03-07T03:09:03.389378+0000","last_became_active":"2024-03-07T03:08:49.454237+0000","last_became_peered":"2024-03-07T03:08:49.454237+0000","last_unstale":"2024-03-07T03:09:03.389378+0000","last_undegraded":"2024-03-07T03:09:03.389378+0000","last_fullsized":"2024-03-07T03:09:03.389378+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:34:35.806144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.383974+0000","last_change":"2024-03-07T03:08:49.422878+0000","last_active":"2024-03-07T03:09:03.383974+0000","last_peered":"2024-03-07T03:09:03.383974+0000","last_clean":"2024-03-07T03:09:03.383974+0000","last_became_active":"2024-03-07T03:08:49.421940+0000","last_became_peered":"2024-03-07T03:08:49.421940+0000","last_unstale":"2024-03-07T03:09:03.383974+0000","last_undegraded":"2024-03-07T03:09:03.383974+0000","last_fullsized":"2024-03-07T03:09:03.383974+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:27:57.145332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.384952+0000","last_change":"2024-03-07T03:08:50.445879+0000","last_active":"2024-03-07T03:09:03.384952+0000","last_peered":"2024-03-07T03:09:03.384952+0000","last_clean":"2024-03-07T03:09:03.384952+0000","last_became_active":"2024-03-07T03:08:50.445045+0000","last_became_peered":"2024-03-07T03:08:50.445045+0000","last_unstale":"2024-03-07T03:09:03.384952+0000","last_undegraded":"2024-03-07T03:09:03.384952+0000","last_fullsized":"2024-03-07T03:09:03.384952+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:43:28.804667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.392269+0000","last_change":"2024-03-07T03:08:49.425773+0000","last_active":"2024-03-07T03:09:03.392269+0000","last_peered":"2024-03-07T03:09:03.392269+0000","last_clean":"2024-03-07T03:09:03.392269+0000","last_became_active":"2024-03-07T03:08:49.424985+0000","last_became_peered":"2024-03-07T03:08:49.424985+0000","last_unstale":"2024-03-07T03:09:03.392269+0000","last_undegraded":"2024-03-07T03:09:03.392269+0000","last_fullsized":"2024-03-07T03:09:03.392269+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:09:51.169893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388370+0000","last_change":"2024-03-07T03:08:49.451452+0000","last_active":"2024-03-07T03:09:03.388370+0000","last_peered":"2024-03-07T03:09:03.388370+0000","last_clean":"2024-03-07T03:09:03.388370+0000","last_became_active":"2024-03-07T03:08:49.450742+0000","last_became_peered":"2024-03-07T03:08:49.450742+0000","last_unstale":"2024-03-07T03:09:03.388370+0000","last_undegraded":"2024-03-07T03:09:03.388370+0000","last_fullsized":"2024-03-07T03:09:03.388370+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:33:02.521002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.390455+0000","last_change":"2024-03-07T03:08:49.414240+0000","last_active":"2024-03-07T03:09:03.390455+0000","last_peered":"2024-03-07T03:09:03.390455+0000","last_clean":"2024-03-07T03:09:03.390455+0000","last_became_active":"2024-03-07T03:08:49.413334+0000","last_became_peered":"2024-03-07T03:08:49.413334+0000","last_unstale":"2024-03-07T03:09:03.390455+0000","last_undegraded":"2024-03-07T03:09:03.390455+0000","last_fullsized":"2024-03-07T03:09:03.390455+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:04:00.717172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.979639+0000","last_change":"2024-03-07T03:06:42.667758+0000","last_active":"2024-03-07T03:09:03.979639+0000","last_peered":"2024-03-07T03:09:03.979639+0000","last_clean":"2024-03-07T03:09:03.979639+0000","last_became_active":"2024-03-07T03:06:42.667171+0000","last_became_peered":"2024-03-07T03:06:42.667171+0000","last_unstale":"2024-03-07T03:09:03.979639+0000","last_undegraded":"2024-03-07T03:09:03.979639+0000","last_fullsized":"2024-03-07T03:09:03.979639+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:55:07.432703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.376990+0000","last_change":"2024-03-07T03:08:50.476783+0000","last_active":"2024-03-07T03:09:03.376990+0000","last_peered":"2024-03-07T03:09:03.376990+0000","last_clean":"2024-03-07T03:09:03.376990+0000","last_became_active":"2024-03-07T03:08:50.476378+0000","last_became_peered":"2024-03-07T03:08:50.476378+0000","last_unstale":"2024-03-07T03:09:03.376990+0000","last_undegraded":"2024-03-07T03:09:03.376990+0000","last_fullsized":"2024-03-07T03:09:03.376990+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:50:41.163577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.387047+0000","last_change":"2024-03-07T03:06:42.794071+0000","last_active":"2024-03-07T03:09:03.387047+0000","last_peered":"2024-03-07T03:09:03.387047+0000","last_clean":"2024-03-07T03:09:03.387047+0000","last_became_active":"2024-03-07T03:06:42.792899+0000","last_became_peered":"2024-03-07T03:06:42.792899+0000","last_unstale":"2024-03-07T03:09:03.387047+0000","last_undegraded":"2024-03-07T03:09:03.387047+0000","last_fullsized":"2024-03-07T03:09:03.387047+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:07:21.918130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.378555+0000","last_change":"2024-03-07T03:08:49.437055+0000","last_active":"2024-03-07T03:09:03.378555+0000","last_peered":"2024-03-07T03:09:03.378555+0000","last_clean":"2024-03-07T03:09:03.378555+0000","last_became_active":"2024-03-07T03:08:49.436547+0000","last_became_peered":"2024-03-07T03:08:49.436547+0000","last_unstale":"2024-03-07T03:09:03.378555+0000","last_undegraded":"2024-03-07T03:09:03.378555+0000","last_fullsized":"2024-03-07T03:09:03.378555+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T15:03:30.849727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.392010+0000","last_change":"2024-03-07T03:08:50.478948+0000","last_active":"2024-03-07T03:09:03.392010+0000","last_peered":"2024-03-07T03:09:03.392010+0000","last_clean":"2024-03-07T03:09:03.392010+0000","last_became_active":"2024-03-07T03:08:50.478021+0000","last_became_peered":"2024-03-07T03:08:50.478021+0000","last_unstale":"2024-03-07T03:09:03.392010+0000","last_undegraded":"2024-03-07T03:09:03.392010+0000","last_fullsized":"2024-03-07T03:09:03.392010+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:14:54.695833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.387469+0000","last_change":"2024-03-07T03:08:49.425684+0000","last_active":"2024-03-07T03:09:03.387469+0000","last_peered":"2024-03-07T03:09:03.387469+0000","last_clean":"2024-03-07T03:09:03.387469+0000","last_became_active":"2024-03-07T03:08:49.424920+0000","last_became_peered":"2024-03-07T03:08:49.424920+0000","last_unstale":"2024-03-07T03:09:03.387469+0000","last_undegraded":"2024-03-07T03:09:03.387469+0000","last_fullsized":"2024-03-07T03:09:03.387469+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:24:18.065220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.379753+0000","last_change":"2024-03-07T03:06:42.792519+0000","last_active":"2024-03-07T03:09:03.379753+0000","last_peered":"2024-03-07T03:09:03.379753+0000","last_clean":"2024-03-07T03:09:03.379753+0000","last_became_active":"2024-03-07T03:06:42.792136+0000","last_became_peered":"2024-03-07T03:06:42.792136+0000","last_unstale":"2024-03-07T03:09:03.379753+0000","last_undegraded":"2024-03-07T03:09:03.379753+0000","last_fullsized":"2024-03-07T03:09:03.379753+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:50:03.686748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.390626+0000","last_change":"2024-03-07T03:08:50.443327+0000","last_active":"2024-03-07T03:09:03.390626+0000","last_peered":"2024-03-07T03:09:03.390626+0000","last_clean":"2024-03-07T03:09:03.390626+0000","last_became_active":"2024-03-07T03:08:50.442549+0000","last_became_peered":"2024-03-07T03:08:50.442549+0000","last_unstale":"2024-03-07T03:09:03.390626+0000","last_undegraded":"2024-03-07T03:09:03.390626+0000","last_fullsized":"2024-03-07T03:09:03.390626+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:31:57.940835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.389951+0000","last_change":"2024-03-07T03:08:49.424192+0000","last_active":"2024-03-07T03:09:03.389951+0000","last_peered":"2024-03-07T03:09:03.389951+0000","last_clean":"2024-03-07T03:09:03.389951+0000","last_became_active":"2024-03-07T03:08:49.421371+0000","last_became_peered":"2024-03-07T03:08:49.421371+0000","last_unstale":"2024-03-07T03:09:03.389951+0000","last_undegraded":"2024-03-07T03:09:03.389951+0000","last_fullsized":"2024-03-07T03:09:03.389951+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:21:00.758921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":22,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:09:04.623474+0000","last_change":"2024-03-07T03:06:42.668416+0000","last_active":"2024-03-07T03:09:04.623474+0000","last_peered":"2024-03-07T03:09:04.623474+0000","last_clean":"2024-03-07T03:09:04.623474+0000","last_became_active":"2024-03-07T03:06:42.668043+0000","last_became_peered":"2024-03-07T03:06:42.668043+0000","last_unstale":"2024-03-07T03:09:04.623474+0000","last_undegraded":"2024-03-07T03:09:04.623474+0000","last_fullsized":"2024-03-07T03:09:04.623474+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:07:59.168239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.979436+0000","last_change":"2024-03-07T03:08:50.484282+0000","last_active":"2024-03-07T03:09:03.979436+0000","last_peered":"2024-03-07T03:09:03.979436+0000","last_clean":"2024-03-07T03:09:03.979436+0000","last_became_active":"2024-03-07T03:08:50.483435+0000","last_became_peered":"2024-03-07T03:08:50.483435+0000","last_unstale":"2024-03-07T03:09:03.979436+0000","last_undegraded":"2024-03-07T03:09:03.979436+0000","last_fullsized":"2024-03-07T03:09:03.979436+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:43:13.608774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377381+0000","last_change":"2024-03-07T03:08:49.423282+0000","last_active":"2024-03-07T03:09:03.377381+0000","last_peered":"2024-03-07T03:09:03.377381+0000","last_clean":"2024-03-07T03:09:03.377381+0000","last_became_active":"2024-03-07T03:08:49.421827+0000","last_became_peered":"2024-03-07T03:08:49.421827+0000","last_unstale":"2024-03-07T03:09:03.377381+0000","last_undegraded":"2024-03-07T03:09:03.377381+0000","last_fullsized":"2024-03-07T03:09:03.377381+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T15:08:27.571976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":269,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:09:04.623234+0000","last_change":"2024-03-07T03:05:08.667040+0000","last_active":"2024-03-07T03:09:04.623234+0000","last_peered":"2024-03-07T03:09:04.623234+0000","last_clean":"2024-03-07T03:09:04.623234+0000","last_became_active":"2024-03-07T03:05:08.665926+0000","last_became_peered":"2024-03-07T03:05:08.665926+0000","last_unstale":"2024-03-07T03:09:04.623234+0000","last_undegraded":"2024-03-07T03:09:04.623234+0000","last_fullsized":"2024-03-07T03:09:04.623234+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:01:00.212577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:01:00.212577+0000","last_clean_scrub_stamp":"2024-03-07T03:01:00.212577+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:11:34.307802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.383577+0000","last_change":"2024-03-07T03:06:42.670236+0000","last_active":"2024-03-07T03:09:03.383577+0000","last_peered":"2024-03-07T03:09:03.383577+0000","last_clean":"2024-03-07T03:09:03.383577+0000","last_became_active":"2024-03-07T03:06:42.669808+0000","last_became_peered":"2024-03-07T03:06:42.669808+0000","last_unstale":"2024-03-07T03:09:03.383577+0000","last_undegraded":"2024-03-07T03:09:03.383577+0000","last_fullsized":"2024-03-07T03:09:03.383577+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:42:36.928386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.387442+0000","last_change":"2024-03-07T03:08:50.440449+0000","last_active":"2024-03-07T03:09:03.387442+0000","last_peered":"2024-03-07T03:09:03.387442+0000","last_clean":"2024-03-07T03:09:03.387442+0000","last_became_active":"2024-03-07T03:08:50.439241+0000","last_became_peered":"2024-03-07T03:08:50.439241+0000","last_unstale":"2024-03-07T03:09:03.387442+0000","last_undegraded":"2024-03-07T03:09:03.387442+0000","last_fullsized":"2024-03-07T03:09:03.387442+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:31:52.265284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.386338+0000","last_change":"2024-03-07T03:08:49.423091+0000","last_active":"2024-03-07T03:09:03.386338+0000","last_peered":"2024-03-07T03:09:03.386338+0000","last_clean":"2024-03-07T03:09:03.386338+0000","last_became_active":"2024-03-07T03:08:49.422236+0000","last_became_peered":"2024-03-07T03:08:49.422236+0000","last_unstale":"2024-03-07T03:09:03.386338+0000","last_undegraded":"2024-03-07T03:09:03.386338+0000","last_fullsized":"2024-03-07T03:09:03.386338+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:16:21.319520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.384189+0000","last_change":"2024-03-07T03:06:42.795963+0000","last_active":"2024-03-07T03:09:03.384189+0000","last_peered":"2024-03-07T03:09:03.384189+0000","last_clean":"2024-03-07T03:09:03.384189+0000","last_became_active":"2024-03-07T03:06:42.795033+0000","last_became_peered":"2024-03-07T03:06:42.795033+0000","last_unstale":"2024-03-07T03:09:03.384189+0000","last_undegraded":"2024-03-07T03:09:03.384189+0000","last_fullsized":"2024-03-07T03:09:03.384189+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:12:44.845051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:09:04.605356+0000","last_change":"2024-03-07T03:09:04.605356+0000","last_active":"2024-03-07T03:09:04.605356+0000","last_peered":"2024-03-07T03:09:04.605356+0000","last_clean":"2024-03-07T03:09:04.605356+0000","last_became_active":"2024-03-07T03:09:04.604227+0000","last_became_peered":"2024-03-07T03:09:04.604227+0000","last_unstale":"2024-03-07T03:09:04.605356+0000","last_undegraded":"2024-03-07T03:09:04.605356+0000","last_fullsized":"2024-03-07T03:09:04.605356+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:45:59.752727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388520+0000","last_change":"2024-03-07T03:08:49.433660+0000","last_active":"2024-03-07T03:09:03.388520+0000","last_peered":"2024-03-07T03:09:03.388520+0000","last_clean":"2024-03-07T03:09:03.388520+0000","last_became_active":"2024-03-07T03:08:49.433002+0000","last_became_peered":"2024-03-07T03:08:49.433002+0000","last_unstale":"2024-03-07T03:09:03.388520+0000","last_undegraded":"2024-03-07T03:09:03.388520+0000","last_fullsized":"2024-03-07T03:09:03.388520+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:07:11.391708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.383913+0000","last_change":"2024-03-07T03:06:42.667215+0000","last_active":"2024-03-07T03:09:03.383913+0000","last_peered":"2024-03-07T03:09:03.383913+0000","last_clean":"2024-03-07T03:09:03.383913+0000","last_became_active":"2024-03-07T03:06:42.666675+0000","last_became_peered":"2024-03-07T03:06:42.666675+0000","last_unstale":"2024-03-07T03:09:03.383913+0000","last_undegraded":"2024-03-07T03:09:03.383913+0000","last_fullsized":"2024-03-07T03:09:03.383913+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:35:31.916960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.386441+0000","last_change":"2024-03-07T03:08:50.478421+0000","last_active":"2024-03-07T03:09:03.386441+0000","last_peered":"2024-03-07T03:09:03.386441+0000","last_clean":"2024-03-07T03:09:03.386441+0000","last_became_active":"2024-03-07T03:08:50.477357+0000","last_became_peered":"2024-03-07T03:08:50.477357+0000","last_unstale":"2024-03-07T03:09:03.386441+0000","last_undegraded":"2024-03-07T03:09:03.386441+0000","last_fullsized":"2024-03-07T03:09:03.386441+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:23:13.285272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.384320+0000","last_change":"2024-03-07T03:08:49.427640+0000","last_active":"2024-03-07T03:09:03.384320+0000","last_peered":"2024-03-07T03:09:03.384320+0000","last_clean":"2024-03-07T03:09:03.384320+0000","last_became_active":"2024-03-07T03:08:49.426778+0000","last_became_peered":"2024-03-07T03:08:49.426778+0000","last_unstale":"2024-03-07T03:09:03.384320+0000","last_undegraded":"2024-03-07T03:09:03.384320+0000","last_fullsized":"2024-03-07T03:09:03.384320+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:44:19.233395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.387281+0000","last_change":"2024-03-07T03:06:42.666671+0000","last_active":"2024-03-07T03:09:03.387281+0000","last_peered":"2024-03-07T03:09:03.387281+0000","last_clean":"2024-03-07T03:09:03.387281+0000","last_became_active":"2024-03-07T03:06:42.666402+0000","last_became_peered":"2024-03-07T03:06:42.666402+0000","last_unstale":"2024-03-07T03:09:03.387281+0000","last_undegraded":"2024-03-07T03:09:03.387281+0000","last_fullsized":"2024-03-07T03:09:03.387281+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:28:33.638276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.387618+0000","last_change":"2024-03-07T03:08:50.482545+0000","last_active":"2024-03-07T03:09:03.387618+0000","last_peered":"2024-03-07T03:09:03.387618+0000","last_clean":"2024-03-07T03:09:03.387618+0000","last_became_active":"2024-03-07T03:08:50.481943+0000","last_became_peered":"2024-03-07T03:08:50.481943+0000","last_unstale":"2024-03-07T03:09:03.387618+0000","last_undegraded":"2024-03-07T03:09:03.387618+0000","last_fullsized":"2024-03-07T03:09:03.387618+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:46:06.527309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377109+0000","last_change":"2024-03-07T03:08:49.439651+0000","last_active":"2024-03-07T03:09:03.377109+0000","last_peered":"2024-03-07T03:09:03.377109+0000","last_clean":"2024-03-07T03:09:03.377109+0000","last_became_active":"2024-03-07T03:08:49.438883+0000","last_became_peered":"2024-03-07T03:08:49.438883+0000","last_unstale":"2024-03-07T03:09:03.377109+0000","last_undegraded":"2024-03-07T03:09:03.377109+0000","last_fullsized":"2024-03-07T03:09:03.377109+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:36:54.345992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.389084+0000","last_change":"2024-03-07T03:08:49.449533+0000","last_active":"2024-03-07T03:09:03.389084+0000","last_peered":"2024-03-07T03:09:03.389084+0000","last_clean":"2024-03-07T03:09:03.389084+0000","last_became_active":"2024-03-07T03:08:49.449003+0000","last_became_peered":"2024-03-07T03:08:49.449003+0000","last_unstale":"2024-03-07T03:09:03.389084+0000","last_undegraded":"2024-03-07T03:09:03.389084+0000","last_fullsized":"2024-03-07T03:09:03.389084+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:46:47.411367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388466+0000","last_change":"2024-03-07T03:08:49.424990+0000","last_active":"2024-03-07T03:09:03.388466+0000","last_peered":"2024-03-07T03:09:03.388466+0000","last_clean":"2024-03-07T03:09:03.388466+0000","last_became_active":"2024-03-07T03:08:49.423891+0000","last_became_peered":"2024-03-07T03:08:49.423891+0000","last_unstale":"2024-03-07T03:09:03.388466+0000","last_undegraded":"2024-03-07T03:09:03.388466+0000","last_fullsized":"2024-03-07T03:09:03.388466+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:06:49.623952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:09:04.605354+0000","last_change":"2024-03-07T03:09:04.605354+0000","last_active":"2024-03-07T03:09:04.605354+0000","last_peered":"2024-03-07T03:09:04.605354+0000","last_clean":"2024-03-07T03:09:04.605354+0000","last_became_active":"2024-03-07T03:09:04.604259+0000","last_became_peered":"2024-03-07T03:09:04.604259+0000","last_unstale":"2024-03-07T03:09:04.605354+0000","last_undegraded":"2024-03-07T03:09:04.605354+0000","last_fullsized":"2024-03-07T03:09:04.605354+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:15:28.053303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.979265+0000","last_change":"2024-03-07T03:08:49.440953+0000","last_active":"2024-03-07T03:09:03.979265+0000","last_peered":"2024-03-07T03:09:03.979265+0000","last_clean":"2024-03-07T03:09:03.979265+0000","last_became_active":"2024-03-07T03:08:49.440238+0000","last_became_peered":"2024-03-07T03:08:49.440238+0000","last_unstale":"2024-03-07T03:09:03.979265+0000","last_undegraded":"2024-03-07T03:09:03.979265+0000","last_fullsized":"2024-03-07T03:09:03.979265+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:56:52.219952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.389485+0000","last_change":"2024-03-07T03:08:49.454889+0000","last_active":"2024-03-07T03:09:03.389485+0000","last_peered":"2024-03-07T03:09:03.389485+0000","last_clean":"2024-03-07T03:09:03.389485+0000","last_became_active":"2024-03-07T03:08:49.454425+0000","last_became_peered":"2024-03-07T03:08:49.454425+0000","last_unstale":"2024-03-07T03:09:03.389485+0000","last_undegraded":"2024-03-07T03:09:03.389485+0000","last_fullsized":"2024-03-07T03:09:03.389485+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:00:02.972842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377619+0000","last_change":"2024-03-07T03:08:49.439517+0000","last_active":"2024-03-07T03:09:03.377619+0000","last_peered":"2024-03-07T03:09:03.377619+0000","last_clean":"2024-03-07T03:09:03.377619+0000","last_became_active":"2024-03-07T03:08:49.438707+0000","last_became_peered":"2024-03-07T03:08:49.438707+0000","last_unstale":"2024-03-07T03:09:03.377619+0000","last_undegraded":"2024-03-07T03:09:03.377619+0000","last_fullsized":"2024-03-07T03:09:03.377619+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:53:47.361431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:09:04.604412+0000","last_change":"2024-03-07T03:09:04.604412+0000","last_active":"2024-03-07T03:09:04.604412+0000","last_peered":"2024-03-07T03:09:04.604412+0000","last_clean":"2024-03-07T03:09:04.604412+0000","last_became_active":"2024-03-07T03:09:04.603338+0000","last_became_peered":"2024-03-07T03:09:04.603338+0000","last_unstale":"2024-03-07T03:09:04.604412+0000","last_undegraded":"2024-03-07T03:09:04.604412+0000","last_fullsized":"2024-03-07T03:09:04.604412+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:10:55.020596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.383871+0000","last_change":"2024-03-07T03:08:49.415515+0000","last_active":"2024-03-07T03:09:03.383871+0000","last_peered":"2024-03-07T03:09:03.383871+0000","last_clean":"2024-03-07T03:09:03.383871+0000","last_became_active":"2024-03-07T03:08:49.414747+0000","last_became_peered":"2024-03-07T03:08:49.414747+0000","last_unstale":"2024-03-07T03:09:03.383871+0000","last_undegraded":"2024-03-07T03:09:03.383871+0000","last_fullsized":"2024-03-07T03:09:03.383871+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:41:43.852936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.979123+0000","last_change":"2024-03-07T03:08:50.479999+0000","last_active":"2024-03-07T03:09:03.979123+0000","last_peered":"2024-03-07T03:09:03.979123+0000","last_clean":"2024-03-07T03:09:03.979123+0000","last_became_active":"2024-03-07T03:08:50.479113+0000","last_became_peered":"2024-03-07T03:08:50.479113+0000","last_unstale":"2024-03-07T03:09:03.979123+0000","last_undegraded":"2024-03-07T03:09:03.979123+0000","last_fullsized":"2024-03-07T03:09:03.979123+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:45:24.316898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.379942+0000","last_change":"2024-03-07T03:08:49.421752+0000","last_active":"2024-03-07T03:09:03.379942+0000","last_peered":"2024-03-07T03:09:03.379942+0000","last_clean":"2024-03-07T03:09:03.379942+0000","last_became_active":"2024-03-07T03:08:49.421165+0000","last_became_peered":"2024-03-07T03:08:49.421165+0000","last_unstale":"2024-03-07T03:09:03.379942+0000","last_undegraded":"2024-03-07T03:09:03.379942+0000","last_fullsized":"2024-03-07T03:09:03.379942+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:57:59.044174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388207+0000","last_change":"2024-03-07T03:08:49.454960+0000","last_active":"2024-03-07T03:09:03.388207+0000","last_peered":"2024-03-07T03:09:03.388207+0000","last_clean":"2024-03-07T03:09:03.388207+0000","last_became_active":"2024-03-07T03:08:49.454394+0000","last_became_peered":"2024-03-07T03:08:49.454394+0000","last_unstale":"2024-03-07T03:09:03.388207+0000","last_undegraded":"2024-03-07T03:09:03.388207+0000","last_fullsized":"2024-03-07T03:09:03.388207+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:39:23.039720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.380113+0000","last_change":"2024-03-07T03:08:49.437330+0000","last_active":"2024-03-07T03:09:03.380113+0000","last_peered":"2024-03-07T03:09:03.380113+0000","last_clean":"2024-03-07T03:09:03.380113+0000","last_became_active":"2024-03-07T03:08:49.436540+0000","last_became_peered":"2024-03-07T03:08:49.436540+0000","last_unstale":"2024-03-07T03:09:03.380113+0000","last_undegraded":"2024-03-07T03:09:03.380113+0000","last_fullsized":"2024-03-07T03:09:03.380113+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:45:32.346918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.383501+0000","last_change":"2024-03-07T03:08:50.110958+0000","last_active":"2024-03-07T03:09:03.383501+0000","last_peered":"2024-03-07T03:09:03.383501+0000","last_clean":"2024-03-07T03:09:03.383501+0000","last_became_active":"2024-03-07T03:08:50.110066+0000","last_became_peered":"2024-03-07T03:08:50.110066+0000","last_unstale":"2024-03-07T03:09:03.383501+0000","last_undegraded":"2024-03-07T03:09:03.383501+0000","last_fullsized":"2024-03-07T03:09:03.383501+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:57:28.837409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.380266+0000","last_change":"2024-03-07T03:08:49.428235+0000","last_active":"2024-03-07T03:09:03.380266+0000","last_peered":"2024-03-07T03:09:03.380266+0000","last_clean":"2024-03-07T03:09:03.380266+0000","last_became_active":"2024-03-07T03:08:49.427415+0000","last_became_peered":"2024-03-07T03:08:49.427415+0000","last_unstale":"2024-03-07T03:09:03.380266+0000","last_undegraded":"2024-03-07T03:09:03.380266+0000","last_fullsized":"2024-03-07T03:09:03.380266+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:22:23.578332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.378281+0000","last_change":"2024-03-07T03:08:50.479547+0000","last_active":"2024-03-07T03:09:03.378281+0000","last_peered":"2024-03-07T03:09:03.378281+0000","last_clean":"2024-03-07T03:09:03.378281+0000","last_became_active":"2024-03-07T03:08:50.478729+0000","last_became_peered":"2024-03-07T03:08:50.478729+0000","last_unstale":"2024-03-07T03:09:03.378281+0000","last_undegraded":"2024-03-07T03:09:03.378281+0000","last_fullsized":"2024-03-07T03:09:03.378281+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:20:56.825216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":32,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.979307+0000","last_change":"2024-03-07T03:08:49.418845+0000","last_active":"2024-03-07T03:09:03.979307+0000","last_peered":"2024-03-07T03:09:03.979307+0000","last_clean":"2024-03-07T03:09:03.979307+0000","last_became_active":"2024-03-07T03:08:49.416207+0000","last_became_peered":"2024-03-07T03:08:49.416207+0000","last_unstale":"2024-03-07T03:09:03.979307+0000","last_undegraded":"2024-03-07T03:09:03.979307+0000","last_fullsized":"2024-03-07T03:09:03.979307+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:50:57.585893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.390071+0000","last_change":"2024-03-07T03:08:50.486370+0000","last_active":"2024-03-07T03:09:03.390071+0000","last_peered":"2024-03-07T03:09:03.390071+0000","last_clean":"2024-03-07T03:09:03.390071+0000","last_became_active":"2024-03-07T03:08:50.486026+0000","last_became_peered":"2024-03-07T03:08:50.486026+0000","last_unstale":"2024-03-07T03:09:03.390071+0000","last_undegraded":"2024-03-07T03:09:03.390071+0000","last_fullsized":"2024-03-07T03:09:03.390071+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:22:43.621291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.378983+0000","last_change":"2024-03-07T03:08:49.427715+0000","last_active":"2024-03-07T03:09:03.378983+0000","last_peered":"2024-03-07T03:09:03.378983+0000","last_clean":"2024-03-07T03:09:03.378983+0000","last_became_active":"2024-03-07T03:08:49.426265+0000","last_became_peered":"2024-03-07T03:08:49.426265+0000","last_unstale":"2024-03-07T03:09:03.378983+0000","last_undegraded":"2024-03-07T03:09:03.378983+0000","last_fullsized":"2024-03-07T03:09:03.378983+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:19:02.652125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.384152+0000","last_change":"2024-03-07T03:08:50.111369+0000","last_active":"2024-03-07T03:09:03.384152+0000","last_peered":"2024-03-07T03:09:03.384152+0000","last_clean":"2024-03-07T03:09:03.384152+0000","last_became_active":"2024-03-07T03:08:50.110035+0000","last_became_peered":"2024-03-07T03:08:50.110035+0000","last_unstale":"2024-03-07T03:09:03.384152+0000","last_undegraded":"2024-03-07T03:09:03.384152+0000","last_fullsized":"2024-03-07T03:09:03.384152+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:39:27.353532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.389730+0000","last_change":"2024-03-07T03:08:49.428095+0000","last_active":"2024-03-07T03:09:03.389730+0000","last_peered":"2024-03-07T03:09:03.389730+0000","last_clean":"2024-03-07T03:09:03.389730+0000","last_became_active":"2024-03-07T03:08:49.426894+0000","last_became_peered":"2024-03-07T03:08:49.426894+0000","last_unstale":"2024-03-07T03:09:03.389730+0000","last_undegraded":"2024-03-07T03:09:03.389730+0000","last_fullsized":"2024-03-07T03:09:03.389730+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:35:45.666181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.379632+0000","last_change":"2024-03-07T03:08:50.479728+0000","last_active":"2024-03-07T03:09:03.379632+0000","last_peered":"2024-03-07T03:09:03.379632+0000","last_clean":"2024-03-07T03:09:03.379632+0000","last_became_active":"2024-03-07T03:08:50.477879+0000","last_became_peered":"2024-03-07T03:08:50.477879+0000","last_unstale":"2024-03-07T03:09:03.379632+0000","last_undegraded":"2024-03-07T03:09:03.379632+0000","last_fullsized":"2024-03-07T03:09:03.379632+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:54:21.740424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.379640+0000","last_change":"2024-03-07T03:08:49.427154+0000","last_active":"2024-03-07T03:09:03.379640+0000","last_peered":"2024-03-07T03:09:03.379640+0000","last_clean":"2024-03-07T03:09:03.379640+0000","last_became_active":"2024-03-07T03:08:49.426306+0000","last_became_peered":"2024-03-07T03:08:49.426306+0000","last_unstale":"2024-03-07T03:09:03.379640+0000","last_undegraded":"2024-03-07T03:09:03.379640+0000","last_fullsized":"2024-03-07T03:09:03.379640+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:21:17.304549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":8,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388748+0000","last_change":"2024-03-07T03:08:49.451933+0000","last_active":"2024-03-07T03:09:03.388748+0000","last_peered":"2024-03-07T03:09:03.388748+0000","last_clean":"2024-03-07T03:09:03.388748+0000","last_became_active":"2024-03-07T03:08:49.451466+0000","last_became_peered":"2024-03-07T03:08:49.451466+0000","last_unstale":"2024-03-07T03:09:03.388748+0000","last_undegraded":"2024-03-07T03:09:03.388748+0000","last_fullsized":"2024-03-07T03:09:03.388748+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:59:00.187710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"70'7","reported_seq":29,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.389525+0000","last_change":"2024-03-07T03:08:49.436124+0000","last_active":"2024-03-07T03:09:03.389525+0000","last_peered":"2024-03-07T03:09:03.389525+0000","last_clean":"2024-03-07T03:09:03.389525+0000","last_became_active":"2024-03-07T03:08:49.435086+0000","last_became_peered":"2024-03-07T03:08:49.435086+0000","last_unstale":"2024-03-07T03:09:03.389525+0000","last_undegraded":"2024-03-07T03:09:03.389525+0000","last_fullsized":"2024-03-07T03:09:03.389525+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:17:31.274262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.386340+0000","last_change":"2024-03-07T03:08:50.444679+0000","last_active":"2024-03-07T03:09:03.386340+0000","last_peered":"2024-03-07T03:09:03.386340+0000","last_clean":"2024-03-07T03:09:03.386340+0000","last_became_active":"2024-03-07T03:08:50.443875+0000","last_became_peered":"2024-03-07T03:08:50.443875+0000","last_unstale":"2024-03-07T03:09:03.386340+0000","last_undegraded":"2024-03-07T03:09:03.386340+0000","last_fullsized":"2024-03-07T03:09:03.386340+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:02:19.642787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":22,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.379845+0000","last_change":"2024-03-07T03:08:49.441798+0000","last_active":"2024-03-07T03:09:03.379845+0000","last_peered":"2024-03-07T03:09:03.379845+0000","last_clean":"2024-03-07T03:09:03.379845+0000","last_became_active":"2024-03-07T03:08:49.441261+0000","last_became_peered":"2024-03-07T03:08:49.441261+0000","last_unstale":"2024-03-07T03:09:03.379845+0000","last_undegraded":"2024-03-07T03:09:03.379845+0000","last_fullsized":"2024-03-07T03:09:03.379845+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:48:15.201462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377374+0000","last_change":"2024-03-07T03:08:50.478907+0000","last_active":"2024-03-07T03:09:03.377374+0000","last_peered":"2024-03-07T03:09:03.377374+0000","last_clean":"2024-03-07T03:09:03.377374+0000","last_became_active":"2024-03-07T03:08:50.478574+0000","last_became_peered":"2024-03-07T03:08:50.478574+0000","last_unstale":"2024-03-07T03:09:03.377374+0000","last_undegraded":"2024-03-07T03:09:03.377374+0000","last_fullsized":"2024-03-07T03:09:03.377374+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:23:54.091751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.383719+0000","last_change":"2024-03-07T03:08:49.407857+0000","last_active":"2024-03-07T03:09:03.383719+0000","last_peered":"2024-03-07T03:09:03.383719+0000","last_clean":"2024-03-07T03:09:03.383719+0000","last_became_active":"2024-03-07T03:08:49.407179+0000","last_became_peered":"2024-03-07T03:08:49.407179+0000","last_unstale":"2024-03-07T03:09:03.383719+0000","last_undegraded":"2024-03-07T03:09:03.383719+0000","last_fullsized":"2024-03-07T03:09:03.383719+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:54:49.068083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377124+0000","last_change":"2024-03-07T03:08:50.439815+0000","last_active":"2024-03-07T03:09:03.377124+0000","last_peered":"2024-03-07T03:09:03.377124+0000","last_clean":"2024-03-07T03:09:03.377124+0000","last_became_active":"2024-03-07T03:08:50.439269+0000","last_became_peered":"2024-03-07T03:08:50.439269+0000","last_unstale":"2024-03-07T03:09:03.377124+0000","last_undegraded":"2024-03-07T03:09:03.377124+0000","last_fullsized":"2024-03-07T03:09:03.377124+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:40:19.306830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.390874+0000","last_change":"2024-03-07T03:08:49.428343+0000","last_active":"2024-03-07T03:09:03.390874+0000","last_peered":"2024-03-07T03:09:03.390874+0000","last_clean":"2024-03-07T03:09:03.390874+0000","last_became_active":"2024-03-07T03:08:49.418275+0000","last_became_peered":"2024-03-07T03:08:49.418275+0000","last_unstale":"2024-03-07T03:09:03.390874+0000","last_undegraded":"2024-03-07T03:09:03.390874+0000","last_fullsized":"2024-03-07T03:09:03.390874+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:09:29.312397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.979524+0000","last_change":"2024-03-07T03:08:50.441016+0000","last_active":"2024-03-07T03:09:03.979524+0000","last_peered":"2024-03-07T03:09:03.979524+0000","last_clean":"2024-03-07T03:09:03.979524+0000","last_became_active":"2024-03-07T03:08:50.440186+0000","last_became_peered":"2024-03-07T03:08:50.440186+0000","last_unstale":"2024-03-07T03:09:03.979524+0000","last_undegraded":"2024-03-07T03:09:03.979524+0000","last_fullsized":"2024-03-07T03:09:03.979524+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:59:05.444181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377255+0000","last_change":"2024-03-07T03:08:49.433197+0000","last_active":"2024-03-07T03:09:03.377255+0000","last_peered":"2024-03-07T03:09:03.377255+0000","last_clean":"2024-03-07T03:09:03.377255+0000","last_became_active":"2024-03-07T03:08:49.426232+0000","last_became_peered":"2024-03-07T03:08:49.426232+0000","last_unstale":"2024-03-07T03:09:03.377255+0000","last_undegraded":"2024-03-07T03:09:03.377255+0000","last_fullsized":"2024-03-07T03:09:03.377255+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:48:13.717656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.390194+0000","last_change":"2024-03-07T03:08:50.486810+0000","last_active":"2024-03-07T03:09:03.390194+0000","last_peered":"2024-03-07T03:09:03.390194+0000","last_clean":"2024-03-07T03:09:03.390194+0000","last_became_active":"2024-03-07T03:08:50.486489+0000","last_became_peered":"2024-03-07T03:08:50.486489+0000","last_unstale":"2024-03-07T03:09:03.390194+0000","last_undegraded":"2024-03-07T03:09:03.390194+0000","last_fullsized":"2024-03-07T03:09:03.390194+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:11:50.953686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.386243+0000","last_change":"2024-03-07T03:08:49.442318+0000","last_active":"2024-03-07T03:09:03.386243+0000","last_peered":"2024-03-07T03:09:03.386243+0000","last_clean":"2024-03-07T03:09:03.386243+0000","last_became_active":"2024-03-07T03:08:49.441511+0000","last_became_peered":"2024-03-07T03:08:49.441511+0000","last_unstale":"2024-03-07T03:09:03.386243+0000","last_undegraded":"2024-03-07T03:09:03.386243+0000","last_fullsized":"2024-03-07T03:09:03.386243+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:21:40.063628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.377802+0000","last_change":"2024-03-07T03:08:50.483512+0000","last_active":"2024-03-07T03:09:03.377802+0000","last_peered":"2024-03-07T03:09:03.377802+0000","last_clean":"2024-03-07T03:09:03.377802+0000","last_became_active":"2024-03-07T03:08:50.482850+0000","last_became_peered":"2024-03-07T03:08:50.482850+0000","last_unstale":"2024-03-07T03:09:03.377802+0000","last_undegraded":"2024-03-07T03:09:03.377802+0000","last_fullsized":"2024-03-07T03:09:03.377802+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:44:50.926405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:09:03.388151+0000","last_change":"2024-03-07T03:08:49.435585+0000","last_active":"2024-03-07T03:09:03.388151+0000","last_peered":"2024-03-07T03:09:03.388151+0000","last_clean":"2024-03-07T03:09:03.388151+0000","last_became_active":"2024-03-07T03:08:49.433630+0000","last_became_peered":"2024-03-07T03:08:49.433630+0000","last_unstale":"2024-03-07T03:09:03.388151+0000","last_undegraded":"2024-03-07T03:09:03.388151+0000","last_fullsized":"2024-03-07T03:09:03.388151+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:56:19.003898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":77,"num_read_kb":80,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201351,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28352,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716800,"statfs":{"total":95995035648,"available":95966003200,"internally_reserved":0,"allocated":1359872,"data_stored":916402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.92600000000000005,"15min":0.92100000000000004},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.2410000000000001,"5min":1.524,"15min":1.524},"last":1.0740000000000001},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.95599999999999996,"15min":1.016},"min":{"1min":0.59599999999999997,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.2290000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.86099999999999999}]},{"osd":1,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.89200000000000002,"15min":0.92000000000000004},"min":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.0840000000000001,"5min":1.28,"15min":1.28},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.014,"15min":1.0509999999999999},"min":{"1min":0.64600000000000002,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.3280000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":1}]},{"osd":2,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.0049999999999999,"15min":1.044},"min":{"1min":0.57699999999999996,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.3899999999999999,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":1.0980000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.99399999999999999,"15min":0.98799999999999999},"min":{"1min":0.72699999999999998,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.329,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":1.119}]},{"osd":3,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.91900000000000004,"15min":0.94099999999999995},"min":{"1min":0.59999999999999998,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.169,"5min":1.327,"15min":1.327},"last":0.96699999999999997},{"interface":"front","average":{"1min":1.038,"5min":1.0549999999999999,"15min":1.079},"min":{"1min":0.84299999999999997,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.194,"5min":1.448,"15min":1.448},"last":1.022}]},{"osd":4,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.95799999999999996,"15min":0.93000000000000005},"min":{"1min":0.71899999999999997,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.4810000000000001,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.999,"5min":0.99199999999999999,"15min":0.99199999999999999},"min":{"1min":0.68899999999999995,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.2549999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.88400000000000001}]},{"osd":5,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.0209999999999999,"15min":0.996},"min":{"1min":0.80600000000000005,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.3879999999999999,"5min":1.446,"15min":1.446},"last":0.82999999999999996},{"interface":"front","average":{"1min":1.079,"5min":1.0900000000000001,"15min":1.113},"min":{"1min":0.77400000000000002,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.4359999999999999,"5min":1.554,"15min":1.554},"last":0.94499999999999995}]},{"osd":6,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":1.135,"15min":1.139},"min":{"1min":0.88500000000000001,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.522,"5min":1.522,"15min":1.522},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.135,"15min":1.1240000000000001},"min":{"1min":0.77300000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.4419999999999999,"5min":1.496,"15min":1.496},"last":0.91300000000000003}]}]},{"osd":6,"up_from":47,"seq":201863462993,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28336,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716816,"statfs":{"total":95995035648,"available":95966019584,"internally_reserved":0,"allocated":1343488,"data_stored":915983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.874,"15min":0.91600000000000004},"min":{"1min":0.55000000000000004,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.175,"5min":1.349,"15min":1.349},"last":1.1830000000000001},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.91600000000000004,"15min":0.91500000000000004},"min":{"1min":0.64600000000000002,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.29,"5min":1.496,"15min":1.496},"last":0.75800000000000001}]},{"osd":1,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.94599999999999995,"15min":0.91800000000000004},"min":{"1min":0.58299999999999996,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":1.2729999999999999,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.149},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.93700000000000006,"15min":0.93400000000000005},"min":{"1min":0.64900000000000002,"5min":0.438,"15min":0.438},"max":{"1min":1.2130000000000001,"5min":1.454,"15min":1.4570000000000001},"last":1.085}]},{"osd":2,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.94399999999999995,"15min":0.91400000000000003},"min":{"1min":0.68100000000000005,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.0089999999999999,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":1.0029999999999999},{"interface":"front","average":{"1min":1.042,"5min":1.006,"15min":0.97399999999999998},"min":{"1min":0.88700000000000001,"5min":0.59399999999999997,"15min":0.55600000000000005},"max":{"1min":1.1699999999999999,"5min":3.6360000000000001,"15min":3.6360000000000001},"last":1.103}]},{"osd":3,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.0009999999999999,"15min":0.96099999999999997},"min":{"1min":0.68500000000000005,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.1950000000000001,"5min":3.9169999999999998,"15min":3.9169999999999998},"last":1.054},{"interface":"front","average":{"1min":1.038,"5min":1.042,"15min":1.012},"min":{"1min":0.83299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.264,"5min":3.4790000000000001,"15min":3.4790000000000001},"last":1.2410000000000001}]},{"osd":4,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.94099999999999995,"15min":0.84599999999999997},"min":{"1min":0.68600000000000005,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.387,"5min":2.0339999999999998,"15min":2.0339999999999998},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.099,"5min":1.008,"15min":0.92200000000000004},"min":{"1min":0.86799999999999999,"5min":0.307,"15min":0.307},"max":{"1min":1.409,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.302}]},{"osd":5,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.0369999999999999,"15min":0.93500000000000005},"min":{"1min":0.80000000000000004,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.248,"5min":2.1720000000000002,"15min":2.1720000000000002},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.087,"5min":1.0529999999999999,"15min":0.98899999999999999},"min":{"1min":0.78600000000000003,"5min":0.374,"15min":0.374},"max":{"1min":1.3859999999999999,"5min":1.587,"15min":1.587},"last":1.2110000000000001}]},{"osd":7,"last update":"Thu Mar 7 03:10:16 2024","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.099,"15min":1.0409999999999999},"min":{"1min":0.96799999999999997,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.4790000000000001,"5min":1.6659999999999999,"15min":1.6659999999999999},"last":1.262},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.1479999999999999,"15min":1.1000000000000001},"min":{"1min":0.74399999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.369,"5min":3.5600000000000001,"15min":3.5600000000000001},"last":1.2829999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574979,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27748,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717404,"statfs":{"total":95995035648,"available":95966621696,"internally_reserved":0,"allocated":741376,"data_stored":324585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.68999999999999995,"15min":0.61799999999999999},"min":{"1min":0.38300000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.95099999999999996,"5min":1.4099999999999999,"15min":1.4099999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.77800000000000002,"15min":0.65800000000000003},"min":{"1min":0.372,"5min":0.34699999999999998,"15min":0.28899999999999998},"max":{"1min":1.244,"5min":1.244,"15min":1.2569999999999999},"last":0.88300000000000001}]},{"osd":2,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.76800000000000002,"15min":0.69399999999999995},"min":{"1min":0.42999999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.1970000000000001,"5min":1.1970000000000001,"15min":1.3169999999999999},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.78800000000000003,"5min":0.78600000000000003,"15min":0.68300000000000005},"min":{"1min":0.51700000000000002,"5min":0.308,"15min":0.308},"max":{"1min":1.097,"5min":1.327,"15min":1.327},"last":0.84399999999999997}]},{"osd":3,"last update":"Thu Mar 7 03:10:13 2024","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.84699999999999998,"15min":0.82099999999999995},"min":{"1min":0.65000000000000002,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.2529999999999999,"5min":1.403,"15min":1.403},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.79100000000000004,"15min":0.78600000000000003},"min":{"1min":0.63400000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.103,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.92000000000000004}]},{"osd":4,"last update":"Thu Mar 7 03:09:57 2024","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":0.97499999999999998,"15min":0.94699999999999995},"min":{"1min":0.79100000000000004,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.2929999999999999,"5min":1.53,"15min":1.53},"last":1.3999999999999999},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0109999999999999,"15min":0.94199999999999995},"min":{"1min":0.76900000000000002,"5min":0.503,"15min":0.503},"max":{"1min":1.3100000000000001,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":1.1599999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:10:41 2024","interfaces":[{"interface":"back","average":{"1min":1,"5min":0.999,"15min":0.96899999999999997},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.3480000000000001,"5min":1.498,"15min":1.498},"last":1.2669999999999999},{"interface":"front","average":{"1min":1.0800000000000001,"5min":1.0600000000000001,"15min":1.0489999999999999},"min":{"1min":0.66700000000000004,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.407,"5min":1.4470000000000001,"15min":1.5600000000000001},"last":1.323}]},{"osd":6,"last update":"Thu Mar 7 03:10:25 2024","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0720000000000001,"15min":1.0840000000000001},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.57799999999999996},"max":{"1min":1.393,"5min":1.5109999999999999,"15min":1.8520000000000001},"last":1.0960000000000001},{"interface":"front","average":{"1min":1.069,"5min":1.0760000000000001,"15min":1.1040000000000001},"min":{"1min":0.72699999999999998,"5min":0.65800000000000003,"15min":0.60899999999999999},"max":{"1min":1.2629999999999999,"5min":1.4990000000000001,"15min":1.512},"last":1.1759999999999999}]},{"osd":7,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.137,"15min":1.175},"min":{"1min":0.75600000000000001,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.4770000000000001,"5min":1.619,"15min":1.619},"last":1.4870000000000001},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.1479999999999999,"15min":1.1399999999999999},"min":{"1min":0.94199999999999995,"5min":0.622,"15min":0.622},"max":{"1min":1.508,"5min":1.508,"15min":1.508},"last":1.375}]}]},{"osd":0,"up_from":8,"seq":34359738509,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28332,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716820,"statfs":{"total":95995035648,"available":95966023680,"internally_reserved":0,"allocated":1343488,"data_stored":915557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Mar 7 03:10:33 2024","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.71699999999999997,"15min":0.623},"min":{"1min":0.42899999999999999,"5min":0.34300000000000003,"15min":0.25},"max":{"1min":1.073,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.753},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.75,"15min":0.67200000000000004},"min":{"1min":0.442,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.1339999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.79100000000000004}]},{"osd":2,"last update":"Thu Mar 7 03:10:05 2024","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.79100000000000004,"15min":0.70399999999999996},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.28499999999999998},"max":{"1min":1.2190000000000001,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.79000000000000004,"5min":0.80500000000000005,"15min":0.73699999999999999},"min":{"1min":0.35999999999999999,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.4279999999999999},"last":0.98999999999999999}]},{"osd":3,"last update":"Thu Mar 7 03:10:05 2024","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.86799999999999999,"15min":0.82099999999999995},"min":{"1min":0.49299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.1559999999999999,"5min":1.276,"15min":1.6180000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.872,"5min":0.89000000000000001,"15min":0.83699999999999997},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.316},"max":{"1min":1.175,"5min":1.5649999999999999,"15min":1.5649999999999999},"last":0.90400000000000003}]},{"osd":4,"last update":"Thu Mar 7 03:09:55 2024","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.0089999999999999,"15min":0.96699999999999997},"min":{"1min":0.75800000000000001,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.506,"5min":1.506,"15min":1.506},"last":1.069},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.92900000000000005,"15min":0.91400000000000003},"min":{"1min":0.66200000000000003,"5min":0.498,"15min":0.498},"max":{"1min":1.169,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.94099999999999995}]},{"osd":5,"last update":"Thu Mar 7 03:10:38 2024","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.077,"15min":1.014},"min":{"1min":0.74299999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.3100000000000001,"5min":1.8109999999999999,"15min":1.8109999999999999},"last":1.306},{"interface":"front","average":{"1min":1.05,"5min":1.0700000000000001,"15min":1.02},"min":{"1min":0.58899999999999997,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.4319999999999999,"5min":1.712,"15min":1.712},"last":1.1419999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:10:23 2024","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.02,"15min":1.022},"min":{"1min":0.68100000000000005,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.3080000000000001,"5min":1.6779999999999999,"15min":1.6779999999999999},"last":1.0189999999999999},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.1679999999999999,"15min":1.1910000000000001},"min":{"1min":0.92700000000000005,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.466,"5min":1.8340000000000001,"15min":1.8340000000000001},"last":1.25}]},{"osd":7,"last update":"Thu Mar 7 03:10:20 2024","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":1.0629999999999999,"15min":1.0649999999999999},"min":{"1min":0.73299999999999998,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.3999999999999999,"5min":1.518,"15min":1.518},"last":1.1000000000000001},{"interface":"front","average":{"1min":1.179,"5min":1.181,"15min":1.194},"min":{"1min":0.77800000000000002,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.4350000000000001,"5min":1.752,"15min":1.752},"last":1.3580000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411449,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27752,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717400,"statfs":{"total":95995035648,"available":95966617600,"internally_reserved":0,"allocated":745472,"data_stored":324928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:13 2024","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.70799999999999996,"15min":0.63800000000000001},"min":{"1min":0.33800000000000002,"5min":0.318,"15min":0.307},"max":{"1min":1.169,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.76900000000000002,"15min":0.68500000000000005},"min":{"1min":0.375,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.0640000000000001,"5min":1.431,"15min":1.431},"last":0.59099999999999997}]},{"osd":1,"last update":"Thu Mar 7 03:10:13 2024","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.71799999999999997,"15min":0.68400000000000005},"min":{"1min":0.47199999999999998,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":0.90200000000000002,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.79200000000000004,"15min":0.72499999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.28999999999999998},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.70899999999999996}]},{"osd":3,"last update":"Thu Mar 7 03:10:13 2024","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.83199999999999996,"15min":0.81699999999999995},"min":{"1min":0.443,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.089,"5min":1.222,"15min":1.2509999999999999},"last":0.94199999999999995},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.84899999999999998,"15min":0.81799999999999995},"min":{"1min":0.41199999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.2110000000000001,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":0.84899999999999998}]},{"osd":4,"last update":"Thu Mar 7 03:09:50 2024","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.0049999999999999,"15min":0.96499999999999997},"min":{"1min":0.72799999999999998,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.256,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":0.98799999999999999},{"interface":"front","average":{"1min":1.024,"5min":0.995,"15min":0.93799999999999994},"min":{"1min":0.746,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.3799999999999999,"5min":1.649,"15min":1.649},"last":0.80500000000000005}]},{"osd":5,"last update":"Thu Mar 7 03:10:42 2024","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.0349999999999999,"15min":0.99299999999999999},"min":{"1min":0.622,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.365,"5min":1.365,"15min":1.4510000000000001},"last":1.056},{"interface":"front","average":{"1min":1.032,"5min":1.04,"15min":1.002},"min":{"1min":0.68100000000000005,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.3779999999999999,"5min":1.53,"15min":1.53},"last":0.91100000000000003}]},{"osd":6,"last update":"Thu Mar 7 03:10:27 2024","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.052,"15min":1.0740000000000001},"min":{"1min":0.64900000000000002,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.341,"5min":1.599,"15min":1.599},"last":1.1279999999999999},{"interface":"front","average":{"1min":1.161,"5min":1.139,"15min":1.127},"min":{"1min":0.77200000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.5589999999999999,"5min":1.746,"15min":1.746},"last":1.2629999999999999}]},{"osd":7,"last update":"Thu Mar 7 03:10:13 2024","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.111,"15min":1.099},"min":{"1min":0.63700000000000001,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.3720000000000001,"5min":1.6759999999999999,"15min":1.6759999999999999},"last":1.1100000000000001},{"interface":"front","average":{"1min":1.133,"5min":1.1639999999999999,"15min":1.1579999999999999},"min":{"1min":0.64000000000000001,"5min":0.501,"15min":0.501},"max":{"1min":1.423,"5min":1.696,"15min":1.696},"last":1.2050000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149805,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27748,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717404,"statfs":{"total":95995035648,"available":95966621696,"internally_reserved":0,"allocated":741376,"data_stored":324402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.67700000000000005,"15min":0.68400000000000005},"min":{"1min":0.33400000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.0800000000000001,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.74299999999999999,"15min":0.748},"min":{"1min":0.38700000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.7110000000000001,"5min":1.7110000000000001,"15min":1.7110000000000001},"last":0.93700000000000006}]},{"osd":1,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.77100000000000002,"15min":0.78500000000000003},"min":{"1min":0.439,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.1559999999999999,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.76200000000000001,"15min":0.76000000000000001},"min":{"1min":0.42699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.1339999999999999,"5min":1.476,"15min":1.476},"last":0.73399999999999999}]},{"osd":2,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.81899999999999995,"15min":0.82099999999999995},"min":{"1min":0.371,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.1559999999999999,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.86599999999999999,"5min":0.79500000000000004,"15min":0.78000000000000003},"min":{"1min":0.40999999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.5269999999999999,"5min":1.5269999999999999,"15min":1.5269999999999999},"last":0.85799999999999998}]},{"osd":4,"last update":"Thu Mar 7 03:09:55 2024","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.97199999999999998,"15min":0.91200000000000003},"min":{"1min":0.75700000000000001,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.452,"5min":1.452,"15min":1.452},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.0209999999999999,"5min":0.99399999999999999,"15min":0.95699999999999996},"min":{"1min":0.84999999999999998,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.2729999999999999,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":1.2749999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:09:42 2024","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":0.97899999999999998,"15min":0.97099999999999997},"min":{"1min":0.66300000000000003,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.97399999999999998},{"interface":"front","average":{"1min":1.012,"5min":1.03,"15min":1.0740000000000001},"min":{"1min":0.85599999999999998,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.2889999999999999,"5min":1.4590000000000001,"15min":1.492},"last":1.0289999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:10:31 2024","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.054,"15min":1.0700000000000001},"min":{"1min":0.98099999999999998,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.4419999999999999,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":1.3500000000000001},{"interface":"front","average":{"1min":1.099,"5min":1.0700000000000001,"15min":1.0660000000000001},"min":{"1min":0.90200000000000002,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.3129999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":1.095}]},{"osd":7,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.0629999999999999,"15min":1.097},"min":{"1min":0.63100000000000001,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.4139999999999999,"5min":1.45,"15min":1.45},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.113,"15min":1.125},"min":{"1min":0.69899999999999995,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.3260000000000001,"5min":1.512,"15min":1.512},"last":1.1990000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986276,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":761856,"data_stored":326022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:09:47 2024","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.84399999999999997,"15min":0.85599999999999998},"min":{"1min":0.51800000000000002,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.105,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.97699999999999998},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.90500000000000003,"15min":0.86799999999999999},"min":{"1min":0.68100000000000005,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.29,"5min":1.2909999999999999,"15min":1.323},"last":1.0800000000000001}]},{"osd":1,"last update":"Thu Mar 7 03:09:47 2024","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.86899999999999999,"15min":0.86899999999999999},"min":{"1min":0.626,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.135,"5min":1.194,"15min":1.194},"last":0.75700000000000001},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.96299999999999997,"15min":0.91000000000000003},"min":{"1min":0.72599999999999998,"5min":0.52700000000000002,"15min":0.498},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":0.91500000000000004}]},{"osd":2,"last update":"Thu Mar 7 03:09:47 2024","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.006,"15min":0.92700000000000005},"min":{"1min":0.78300000000000003,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.3540000000000001,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.91500000000000004,"15min":0.87},"min":{"1min":0.77600000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.302,"5min":1.302,"15min":1.3720000000000001},"last":0.90000000000000002}]},{"osd":3,"last update":"Thu Mar 7 03:09:47 2024","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":0.94699999999999995,"15min":0.90800000000000003},"min":{"1min":0.77000000000000002,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.2569999999999999,"5min":1.2849999999999999,"15min":1.286},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.878,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.61699999999999999,"5min":0.504,"15min":0.504},"max":{"1min":1.1040000000000001,"5min":1.27,"15min":1.27},"last":0.86299999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:10:35 2024","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.93200000000000005,"15min":0.78300000000000003},"min":{"1min":0.67000000000000004,"5min":0.40000000000000002,"15min":0.33700000000000002},"max":{"1min":1.292,"5min":1.3109999999999999,"15min":1.349},"last":1.131},{"interface":"front","average":{"1min":1.014,"5min":0.93300000000000005,"15min":0.80800000000000005},"min":{"1min":0.71999999999999997,"5min":0.41099999999999998,"15min":0.374},"max":{"1min":1.3520000000000001,"5min":1.385,"15min":1.385},"last":0.94999999999999996}]},{"osd":6,"last update":"Thu Mar 7 03:10:28 2024","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":0.97299999999999998,"15min":0.89000000000000001},"min":{"1min":0.89600000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.2909999999999999,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":0.997},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.069,"15min":0.97899999999999998},"min":{"1min":0.73399999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.492,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.179}]},{"osd":7,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.089,"15min":1.0620000000000001},"min":{"1min":0.94699999999999995,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.357,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":1.222},{"interface":"front","average":{"1min":1.052,"5min":1.0009999999999999,"15min":0.97899999999999998},"min":{"1min":0.85899999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.264,"5min":1.419,"15min":1.419},"last":1.0469999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724634,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27752,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717400,"statfs":{"total":95995035648,"available":95966617600,"internally_reserved":0,"allocated":745472,"data_stored":324906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:41 2024","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.89400000000000002,"15min":0.84599999999999997},"min":{"1min":0.53200000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.2809999999999999,"5min":1.5189999999999999,"15min":1.665},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.92000000000000004,"15min":0.86899999999999999},"min":{"1min":0.70899999999999996,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.2190000000000001,"5min":2.359,"15min":2.359},"last":1.2190000000000001}]},{"osd":1,"last update":"Thu Mar 7 03:10:41 2024","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.93100000000000005,"15min":0.88200000000000001},"min":{"1min":0.65100000000000002,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.179,"5min":1.262,"15min":1.276},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.91200000000000003,"15min":0.88500000000000001},"min":{"1min":0.55500000000000005,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.244,"5min":2.0169999999999999,"15min":2.0169999999999999},"last":0.996}]},{"osd":2,"last update":"Thu Mar 7 03:10:41 2024","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.92000000000000004,"15min":0.93700000000000006},"min":{"1min":0.61799999999999999,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.2729999999999999,"5min":1.9610000000000001,"15min":1.9610000000000001},"last":0.77800000000000002},{"interface":"front","average":{"1min":1.018,"5min":0.98399999999999999,"15min":0.91400000000000003},"min":{"1min":0.77200000000000002,"5min":0.438,"15min":0.438},"max":{"1min":1.274,"5min":1.9770000000000001,"15min":1.9770000000000001},"last":1.091}]},{"osd":3,"last update":"Thu Mar 7 03:10:41 2024","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.97299999999999998,"15min":0.93700000000000006},"min":{"1min":0.70999999999999996,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.22,"5min":1.419,"15min":1.419},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.044,"5min":1.008,"15min":0.96399999999999997},"min":{"1min":0.75700000000000001,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.3160000000000001,"5min":2.0720000000000001,"15min":2.0720000000000001},"last":1.1839999999999999}]},{"osd":4,"last update":"Thu Mar 7 03:10:41 2024","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.86899999999999999,"15min":0.82499999999999996},"min":{"1min":0.63600000000000001,"5min":0.373,"15min":0.35299999999999998},"max":{"1min":1.1919999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":1.145},{"interface":"front","average":{"1min":1.038,"5min":0.99099999999999999,"15min":0.88800000000000001},"min":{"1min":0.77700000000000002,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.2210000000000001,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":1.0569999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:10:30 2024","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.0069999999999999,"15min":0.98799999999999999},"min":{"1min":0.84699999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.351,"5min":1.546,"15min":1.546},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.0229999999999999,"15min":0.995},"min":{"1min":0.66600000000000004,"5min":0.379,"15min":0.379},"max":{"1min":1.401,"5min":1.627,"15min":1.627},"last":1.2430000000000001}]},{"osd":7,"last update":"Thu Mar 7 03:10:12 2024","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0189999999999999,"15min":0.96499999999999997},"min":{"1min":0.73999999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.355,"5min":1.51,"15min":1.51},"last":1.266},{"interface":"front","average":{"1min":1.157,"5min":1.1120000000000001,"15min":1.0349999999999999},"min":{"1min":0.89400000000000002,"5min":0.435,"15min":0.435},"max":{"1min":1.373,"5min":1.782,"15min":1.782},"last":1.121}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-07T03:10:47.089 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-07T03:10:47.089 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-07T03:10:47.089 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-07T03:10:47.090 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:10:47.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:47 smithi145 bash[21339]: audit 2024-03-07T03:10:46.275754+0000 mgr.y (mgr.24302) 183 : audit 0 from='client.24641 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:10:47.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:47 smithi145 bash[30906]: audit 2024-03-07T03:10:46.275754+0000 mgr.y (mgr.24302) 183 : audit 0 from='client.24641 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:10:47.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:47 smithi189 bash[23614]: audit 2024-03-07T03:10:46.275754+0000 mgr.y (mgr.24302) 183 : audit 0 from='client.24641 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:10:48.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:48 smithi145 bash[21339]: cluster 2024-03-07T03:10:46.447915+0000 mgr.y (mgr.24302) 184 : cluster 0 pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:48.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:48 smithi145 bash[21339]: audit 2024-03-07T03:10:47.452481+0000 mon.c (mon.2) 115 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:48.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:48 smithi145 bash[30906]: cluster 2024-03-07T03:10:46.447915+0000 mgr.y (mgr.24302) 184 : cluster 0 pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:48.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:48 smithi145 bash[30906]: audit 2024-03-07T03:10:47.452481+0000 mon.c (mon.2) 115 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:48.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:48 smithi189 bash[23614]: cluster 2024-03-07T03:10:46.447915+0000 mgr.y (mgr.24302) 184 : cluster 0 pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:48.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:48 smithi189 bash[23614]: audit 2024-03-07T03:10:47.452481+0000 mon.c (mon.2) 115 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:49.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:49 smithi145 bash[21339]: cluster 2024-03-07T03:10:48.448503+0000 mgr.y (mgr.24302) 185 : cluster 0 pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:49.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:49 smithi145 bash[30906]: cluster 2024-03-07T03:10:48.448503+0000 mgr.y (mgr.24302) 185 : cluster 0 pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:49.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:49 smithi189 bash[23614]: cluster 2024-03-07T03:10:48.448503+0000 mgr.y (mgr.24302) 185 : cluster 0 pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:51.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:51 smithi145 bash[21339]: cluster 2024-03-07T03:10:50.449897+0000 mgr.y (mgr.24302) 186 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:51.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:51 smithi145 bash[30906]: cluster 2024-03-07T03:10:50.449897+0000 mgr.y (mgr.24302) 186 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:51.821 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:51.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:51 smithi189 bash[23614]: cluster 2024-03-07T03:10:50.449897+0000 mgr.y (mgr.24302) 186 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:52.812 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:10:52 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:10:52] "GET /metrics HTTP/1.1" 200 33766 "" "Prometheus/2.43.0" 2024-03-07T03:10:54.729 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:54 smithi145 bash[21339]: cluster 2024-03-07T03:10:52.450535+0000 mgr.y (mgr.24302) 187 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:54.730 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:54 smithi145 bash[30906]: cluster 2024-03-07T03:10:52.450535+0000 mgr.y (mgr.24302) 187 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:54.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:54 smithi189 bash[23614]: cluster 2024-03-07T03:10:52.450535+0000 mgr.y (mgr.24302) 187 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:55.144 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:10:55.145 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-07T03:10:55.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:55 smithi145 bash[21339]: audit 2024-03-07T03:10:55.154474+0000 mon.a (mon.0) 845 : audit 0 from='client.? 172.21.15.145:0/3263463814' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:10:55.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:55 smithi145 bash[30906]: audit 2024-03-07T03:10:55.154474+0000 mon.a (mon.0) 845 : audit 0 from='client.? 172.21.15.145:0/3263463814' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:10:55.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:55 smithi189 bash[23614]: audit 2024-03-07T03:10:55.154474+0000 mon.a (mon.0) 845 : audit 0 from='client.? 172.21.15.145:0/3263463814' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:10:56.001 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-07T03:10:56.001 INFO:teuthology.run_tasks:Running task print... 2024-03-07T03:10:56.011 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-03-07T03:10:56.011 INFO:teuthology.run_tasks:Running task parallel... 2024-03-07T03:10:56.059 INFO:teuthology.task.parallel:starting parallel... 2024-03-07T03:10:56.059 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-07T03:10:56.060 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-03-07T03:10:56.063 INFO:tasks.workunit:Pulling workunits from ref reef 2024-03-07T03:10:56.063 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-03-07T03:10:56.064 DEBUG:teuthology.orchestra.run.smithi145:> stat -- /home/ubuntu/cephtest/mnt.0 2024-03-07T03:10:56.064 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-07T03:10:56.064 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-03-07T03:10:56.065 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi145.front.sepia.ceph.com 2024-03-07T03:10:56.065 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-03-07T03:10:56.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-07T03:10:56.071 INFO:teuthology.orchestra.run.smithi145.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-03-07T03:10:56.071 DEBUG:teuthology.orchestra.run.smithi145:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-03-07T03:10:56.117 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-03-07T03:10:56.117 DEBUG:teuthology.orchestra.run.smithi145:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-03-07T03:10:56.165 INFO:tasks.workunit:timeout=3h 2024-03-07T03:10:56.165 INFO:tasks.workunit:cleanup=True 2024-03-07T03:10:56.165 DEBUG:teuthology.orchestra.run.smithi145:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-03-07T03:10:56.212 INFO:tasks.workunit.client.0.smithi145.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-03-07T03:10:56.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:56 smithi145 bash[21339]: cluster 2024-03-07T03:10:54.451591+0000 mgr.y (mgr.24302) 188 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:56.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:56 smithi145 bash[30906]: cluster 2024-03-07T03:10:54.451591+0000 mgr.y (mgr.24302) 188 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:56.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:56 smithi189 bash[23614]: cluster 2024-03-07T03:10:54.451591+0000 mgr.y (mgr.24302) 188 : cluster 0 pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:58.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:10:58 smithi145 bash[21339]: cluster 2024-03-07T03:10:56.452334+0000 mgr.y (mgr.24302) 189 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:58.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:10:58 smithi145 bash[30906]: cluster 2024-03-07T03:10:56.452334+0000 mgr.y (mgr.24302) 189 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:58.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:10:58 smithi189 bash[23614]: cluster 2024-03-07T03:10:56.452334+0000 mgr.y (mgr.24302) 189 : cluster 0 pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:00.573 DEBUG:teuthology.orchestra.run.smithi145:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-03-07T03:11:00.624 INFO:tasks.workunit.client.0.smithi145.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-03-07T03:11:00.628 INFO:tasks.workunit.client.0.smithi145.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-03-07T03:11:00.628 INFO:tasks.workunit.client.0.smithi145.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-03-07T03:11:00.786 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:11:00.807 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:00 smithi145 bash[21339]: cluster 2024-03-07T03:10:58.452766+0000 mgr.y (mgr.24302) 190 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:00.807 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:00 smithi145 bash[30906]: cluster 2024-03-07T03:10:58.452766+0000 mgr.y (mgr.24302) 190 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:00.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:00 smithi189 bash[23614]: cluster 2024-03-07T03:10:58.452766+0000 mgr.y (mgr.24302) 190 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:01.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:01 smithi145 bash[21339]: cluster 2024-03-07T03:11:00.453758+0000 mgr.y (mgr.24302) 191 : cluster 0 pgmap v164: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:01.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:01 smithi145 bash[30906]: cluster 2024-03-07T03:11:00.453758+0000 mgr.y (mgr.24302) 191 : cluster 0 pgmap v164: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:01.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:01 smithi189 bash[23614]: cluster 2024-03-07T03:11:00.453758+0000 mgr.y (mgr.24302) 191 : cluster 0 pgmap v164: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:02.812 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:11:02 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:11:02] "GET /metrics HTTP/1.1" 200 33766 "" "Prometheus/2.43.0" 2024-03-07T03:11:02.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:02 smithi145 bash[21339]: audit 2024-03-07T03:11:02.453021+0000 mon.c (mon.2) 116 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:02.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:02 smithi145 bash[30906]: audit 2024-03-07T03:11:02.453021+0000 mon.c (mon.2) 116 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:02.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:02 smithi189 bash[23614]: audit 2024-03-07T03:11:02.453021+0000 mon.c (mon.2) 116 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:03.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:03 smithi145 bash[21339]: cluster 2024-03-07T03:11:02.454426+0000 mgr.y (mgr.24302) 192 : cluster 0 pgmap v165: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:03.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:03 smithi145 bash[21339]: audit 2024-03-07T03:11:02.561839+0000 mon.c (mon.2) 117 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:03.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:03 smithi145 bash[21339]: audit 2024-03-07T03:11:03.192842+0000 mon.c (mon.2) 118 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:03.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:03 smithi145 bash[21339]: audit 2024-03-07T03:11:03.194774+0000 mon.c (mon.2) 119 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:03.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:03 smithi145 bash[21339]: audit 2024-03-07T03:11:03.205524+0000 mon.a (mon.0) 846 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:03.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:03 smithi145 bash[30906]: cluster 2024-03-07T03:11:02.454426+0000 mgr.y (mgr.24302) 192 : cluster 0 pgmap v165: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:03 smithi145 bash[30906]: audit 2024-03-07T03:11:02.561839+0000 mon.c (mon.2) 117 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:03 smithi145 bash[30906]: audit 2024-03-07T03:11:03.192842+0000 mon.c (mon.2) 118 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:03 smithi145 bash[30906]: audit 2024-03-07T03:11:03.194774+0000 mon.c (mon.2) 119 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:03 smithi145 bash[30906]: audit 2024-03-07T03:11:03.205524+0000 mon.a (mon.0) 846 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:03.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:03 smithi189 bash[23614]: cluster 2024-03-07T03:11:02.454426+0000 mgr.y (mgr.24302) 192 : cluster 0 pgmap v165: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:03.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:03 smithi189 bash[23614]: audit 2024-03-07T03:11:02.561839+0000 mon.c (mon.2) 117 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:03.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:03 smithi189 bash[23614]: audit 2024-03-07T03:11:03.192842+0000 mon.c (mon.2) 118 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:03.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:03 smithi189 bash[23614]: audit 2024-03-07T03:11:03.194774+0000 mon.c (mon.2) 119 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:03.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:03 smithi189 bash[23614]: audit 2024-03-07T03:11:03.205524+0000 mon.a (mon.0) 846 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:05.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:05 smithi145 bash[30906]: cluster 2024-03-07T03:11:04.455462+0000 mgr.y (mgr.24302) 193 : cluster 0 pgmap v166: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:05.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:05 smithi145 bash[21339]: cluster 2024-03-07T03:11:04.455462+0000 mgr.y (mgr.24302) 193 : cluster 0 pgmap v166: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:05.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:05 smithi189 bash[23614]: cluster 2024-03-07T03:11:04.455462+0000 mgr.y (mgr.24302) 193 : cluster 0 pgmap v166: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:06.324 INFO:tasks.workunit.client.0.smithi145.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-03-07T03:11:06.828 INFO:tasks.workunit.client.0.smithi145.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-03-07T03:11:07.030 INFO:tasks.workunit.client.0.smithi145.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-03-07T03:11:07.031 INFO:tasks.workunit.client.0.smithi145.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-03-07T03:11:07.031 INFO:tasks.workunit.client.0.smithi145.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-03-07T03:11:07.083 INFO:tasks.workunit.client.0.smithi145.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-03-07T03:11:07.087 DEBUG:teuthology.orchestra.run.smithi145:> set -ex 2024-03-07T03:11:07.087 DEBUG:teuthology.orchestra.run.smithi145:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-03-07T03:11:07.132 INFO:tasks.workunit:Running workunits matching rbd/test_librbd.sh on client.0... 2024-03-07T03:11:07.133 INFO:tasks.workunit:Running workunit rbd/test_librbd.sh... 2024-03-07T03:11:07.133 DEBUG:teuthology.orchestra.run.smithi145:workunit test rbd/test_librbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd.sh 2024-03-07T03:11:07.345 INFO:tasks.workunit.client.0.smithi145.stderr:warning: failed to enable lockdep 2024-03-07T03:11:07.345 INFO:tasks.workunit.client.0.smithi145.stdout:seed 68337 2024-03-07T03:11:07.345 INFO:tasks.workunit.client.0.smithi145.stdout:[==========] Running 476 tests from 24 test suites. 2024-03-07T03:11:07.345 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] Global test environment set-up. 2024-03-07T03:11:07.345 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 1 test from TestJournalStress 2024-03-07T03:11:08.085 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:11:07 smithi145 bash[21623]: [07/Mar/2024:03:11:07] ENGINE Bus STOPPING 2024-03-07T03:11:08.086 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:11:07 smithi145 bash[21623]: [07/Mar/2024:03:11:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:11:08.086 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:11:07 smithi145 bash[21623]: [07/Mar/2024:03:11:07] ENGINE Bus STOPPED 2024-03-07T03:11:08.086 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:11:07 smithi145 bash[21623]: [07/Mar/2024:03:11:07] ENGINE Bus STARTING 2024-03-07T03:11:08.086 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:11:07 smithi145 bash[21623]: [07/Mar/2024:03:11:07] ENGINE Serving on http://:::9283 2024-03-07T03:11:08.086 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:11:07 smithi145 bash[21623]: [07/Mar/2024:03:11:07] ENGINE Bus STARTED 2024-03-07T03:11:08.086 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:07 smithi145 bash[21339]: cluster 2024-03-07T03:11:06.456105+0000 mgr.y (mgr.24302) 194 : cluster 0 pgmap v167: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:08.086 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:07 smithi145 bash[21339]: audit 2024-03-07T03:11:07.833883+0000 mon.a (mon.0) 847 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:11:08.086 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:07 smithi145 bash[21339]: audit 2024-03-07T03:11:07.871064+0000 mon.c (mon.2) 120 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:08.086 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:07 smithi145 bash[30906]: cluster 2024-03-07T03:11:06.456105+0000 mgr.y (mgr.24302) 194 : cluster 0 pgmap v167: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:08.087 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:07 smithi145 bash[30906]: audit 2024-03-07T03:11:07.833883+0000 mon.a (mon.0) 847 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:11:08.087 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:07 smithi145 bash[30906]: audit 2024-03-07T03:11:07.871064+0000 mon.c (mon.2) 120 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:08.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:07 smithi189 bash[23614]: cluster 2024-03-07T03:11:06.456105+0000 mgr.y (mgr.24302) 194 : cluster 0 pgmap v167: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:08.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:07 smithi189 bash[23614]: audit 2024-03-07T03:11:07.833883+0000 mon.a (mon.0) 847 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:11:08.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:07 smithi189 bash[23614]: audit 2024-03-07T03:11:07.871064+0000 mon.c (mon.2) 120 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:08.769 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph config set global log_to_journald false --force' 2024-03-07T03:11:08.886 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalStress.DiscardWithPruneWriteOverlap 2024-03-07T03:11:09.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:08 smithi145 bash[21339]: cluster 2024-03-07T03:11:07.882115+0000 mon.a (mon.0) 848 : cluster 0 osdmap e73: 8 total, 8 up, 8 in 2024-03-07T03:11:09.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:08 smithi145 bash[21339]: audit 2024-03-07T03:11:07.883302+0000 mon.a (mon.0) 849 : audit 1 from='client.? 172.21.15.145:0/778836170' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:09.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:08 smithi145 bash[21339]: audit 2024-03-07T03:11:08.888216+0000 mon.a (mon.0) 850 : audit 1 from='client.? 172.21.15.145:0/778836170' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:09.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:08 smithi145 bash[21339]: cluster 2024-03-07T03:11:08.888796+0000 mon.a (mon.0) 851 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2024-03-07T03:11:09.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:08 smithi145 bash[30906]: cluster 2024-03-07T03:11:07.882115+0000 mon.a (mon.0) 848 : cluster 0 osdmap e73: 8 total, 8 up, 8 in 2024-03-07T03:11:09.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:08 smithi145 bash[30906]: audit 2024-03-07T03:11:07.883302+0000 mon.a (mon.0) 849 : audit 1 from='client.? 172.21.15.145:0/778836170' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:09.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:08 smithi145 bash[30906]: audit 2024-03-07T03:11:08.888216+0000 mon.a (mon.0) 850 : audit 1 from='client.? 172.21.15.145:0/778836170' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:09.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:08 smithi145 bash[30906]: cluster 2024-03-07T03:11:08.888796+0000 mon.a (mon.0) 851 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2024-03-07T03:11:09.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:08 smithi189 bash[23614]: cluster 2024-03-07T03:11:07.882115+0000 mon.a (mon.0) 848 : cluster 0 osdmap e73: 8 total, 8 up, 8 in 2024-03-07T03:11:09.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:08 smithi189 bash[23614]: audit 2024-03-07T03:11:07.883302+0000 mon.a (mon.0) 849 : audit 1 from='client.? 172.21.15.145:0/778836170' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:09.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:08 smithi189 bash[23614]: audit 2024-03-07T03:11:08.888216+0000 mon.a (mon.0) 850 : audit 1 from='client.? 172.21.15.145:0/778836170' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:09.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:08 smithi189 bash[23614]: cluster 2024-03-07T03:11:08.888796+0000 mon.a (mon.0) 851 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2024-03-07T03:11:10.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:09 smithi145 bash[21339]: cluster 2024-03-07T03:11:08.456804+0000 mgr.y (mgr.24302) 195 : cluster 0 pgmap v169: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:10.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:09 smithi145 bash[21339]: cluster 2024-03-07T03:11:09.395225+0000 mon.a (mon.0) 852 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:10.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:09 smithi145 bash[21339]: cluster 2024-03-07T03:11:09.890582+0000 mon.a (mon.0) 853 : cluster 0 osdmap e75: 8 total, 8 up, 8 in 2024-03-07T03:11:10.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:09 smithi145 bash[30906]: cluster 2024-03-07T03:11:08.456804+0000 mgr.y (mgr.24302) 195 : cluster 0 pgmap v169: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:10.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:09 smithi145 bash[30906]: cluster 2024-03-07T03:11:09.395225+0000 mon.a (mon.0) 852 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:10.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:09 smithi145 bash[30906]: cluster 2024-03-07T03:11:09.890582+0000 mon.a (mon.0) 853 : cluster 0 osdmap e75: 8 total, 8 up, 8 in 2024-03-07T03:11:10.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:09 smithi189 bash[23614]: cluster 2024-03-07T03:11:08.456804+0000 mgr.y (mgr.24302) 195 : cluster 0 pgmap v169: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:10.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:09 smithi189 bash[23614]: cluster 2024-03-07T03:11:09.395225+0000 mon.a (mon.0) 852 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:10.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:09 smithi189 bash[23614]: cluster 2024-03-07T03:11:09.890582+0000 mon.a (mon.0) 853 : cluster 0 osdmap e75: 8 total, 8 up, 8 in 2024-03-07T03:11:10.919 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Stress.cc:37: Skipped 2024-03-07T03:11:10.919 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:10.920 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalStress.DiscardWithPruneWriteOverlap (2033 ms) 2024-03-07T03:11:11.916 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 1 test from TestJournalStress (2033 ms total) 2024-03-07T03:11:11.916 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:11:11.916 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 16 tests from TestJournalReplay 2024-03-07T03:11:12.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:11 smithi145 bash[21339]: cluster 2024-03-07T03:11:10.457525+0000 mgr.y (mgr.24302) 196 : cluster 0 pgmap v172: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:12.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:11 smithi145 bash[21339]: cluster 2024-03-07T03:11:10.893681+0000 mon.a (mon.0) 854 : cluster 0 osdmap e76: 8 total, 8 up, 8 in 2024-03-07T03:11:12.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:11 smithi145 bash[21339]: cluster 2024-03-07T03:11:10.909141+0000 mon.a (mon.0) 855 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:12.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:11 smithi145 bash[21339]: cluster 2024-03-07T03:11:10.909198+0000 mon.a (mon.0) 856 : cluster 1 Cluster is now healthy 2024-03-07T03:11:12.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:11 smithi145 bash[30906]: cluster 2024-03-07T03:11:10.457525+0000 mgr.y (mgr.24302) 196 : cluster 0 pgmap v172: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:12.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:11 smithi145 bash[30906]: cluster 2024-03-07T03:11:10.893681+0000 mon.a (mon.0) 854 : cluster 0 osdmap e76: 8 total, 8 up, 8 in 2024-03-07T03:11:12.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:11 smithi145 bash[30906]: cluster 2024-03-07T03:11:10.909141+0000 mon.a (mon.0) 855 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:12.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:11 smithi145 bash[30906]: cluster 2024-03-07T03:11:10.909198+0000 mon.a (mon.0) 856 : cluster 1 Cluster is now healthy 2024-03-07T03:11:12.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:11 smithi189 bash[23614]: cluster 2024-03-07T03:11:10.457525+0000 mgr.y (mgr.24302) 196 : cluster 0 pgmap v172: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:12.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:11 smithi189 bash[23614]: cluster 2024-03-07T03:11:10.893681+0000 mon.a (mon.0) 854 : cluster 0 osdmap e76: 8 total, 8 up, 8 in 2024-03-07T03:11:12.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:11 smithi189 bash[23614]: cluster 2024-03-07T03:11:10.909141+0000 mon.a (mon.0) 855 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:12.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:11 smithi189 bash[23614]: cluster 2024-03-07T03:11:10.909198+0000 mon.a (mon.0) 856 : cluster 1 Cluster is now healthy 2024-03-07T03:11:12.708 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:11:12.728 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:11:12 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:11:12] "GET /metrics HTTP/1.1" 200 33764 "" "Prometheus/2.43.0" 2024-03-07T03:11:13.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:12 smithi145 bash[21339]: cluster 2024-03-07T03:11:11.921069+0000 mon.a (mon.0) 857 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2024-03-07T03:11:13.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:12 smithi145 bash[30906]: cluster 2024-03-07T03:11:11.921069+0000 mon.a (mon.0) 857 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2024-03-07T03:11:13.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:12 smithi189 bash[23614]: cluster 2024-03-07T03:11:11.921069+0000 mon.a (mon.0) 857 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2024-03-07T03:11:13.959 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.AioDiscardEvent 2024-03-07T03:11:14.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[30906]: cluster 2024-03-07T03:11:12.458291+0000 mgr.y (mgr.24302) 197 : cluster 0 pgmap v175: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:14.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[30906]: cluster 2024-03-07T03:11:12.932436+0000 mon.a (mon.0) 858 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2024-03-07T03:11:14.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[30906]: audit 2024-03-07T03:11:12.937798+0000 mon.c (mon.2) 121 : audit 1 from='client.? 172.21.15.145:0/3656278432' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:14.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[30906]: audit 2024-03-07T03:11:12.947145+0000 mon.a (mon.0) 859 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:14.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[30906]: audit 2024-03-07T03:11:13.306708+0000 mon.a (mon.0) 860 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[30906]: audit 2024-03-07T03:11:13.312635+0000 mon.a (mon.0) 861 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[30906]: audit 2024-03-07T03:11:13.618425+0000 mon.a (mon.0) 862 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[30906]: audit 2024-03-07T03:11:13.638039+0000 mon.a (mon.0) 863 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[30906]: audit 2024-03-07T03:11:13.642323+0000 mon.c (mon.2) 122 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:14.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[30906]: audit 2024-03-07T03:11:13.644059+0000 mon.c (mon.2) 123 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:14.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[30906]: audit 2024-03-07T03:11:13.659218+0000 mon.a (mon.0) 864 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[21339]: cluster 2024-03-07T03:11:12.458291+0000 mgr.y (mgr.24302) 197 : cluster 0 pgmap v175: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:14.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[21339]: cluster 2024-03-07T03:11:12.932436+0000 mon.a (mon.0) 858 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2024-03-07T03:11:14.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[21339]: audit 2024-03-07T03:11:12.937798+0000 mon.c (mon.2) 121 : audit 1 from='client.? 172.21.15.145:0/3656278432' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:14.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[21339]: audit 2024-03-07T03:11:12.947145+0000 mon.a (mon.0) 859 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:14.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[21339]: audit 2024-03-07T03:11:13.306708+0000 mon.a (mon.0) 860 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[21339]: audit 2024-03-07T03:11:13.312635+0000 mon.a (mon.0) 861 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[21339]: audit 2024-03-07T03:11:13.618425+0000 mon.a (mon.0) 862 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[21339]: audit 2024-03-07T03:11:13.638039+0000 mon.a (mon.0) 863 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[21339]: audit 2024-03-07T03:11:13.642323+0000 mon.c (mon.2) 122 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:14.314 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[21339]: audit 2024-03-07T03:11:13.644059+0000 mon.c (mon.2) 123 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:14.314 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:13 smithi145 bash[21339]: audit 2024-03-07T03:11:13.659218+0000 mon.a (mon.0) 864 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:13 smithi189 bash[23614]: cluster 2024-03-07T03:11:12.458291+0000 mgr.y (mgr.24302) 197 : cluster 0 pgmap v175: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:14.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:13 smithi189 bash[23614]: cluster 2024-03-07T03:11:12.932436+0000 mon.a (mon.0) 858 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2024-03-07T03:11:14.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:13 smithi189 bash[23614]: audit 2024-03-07T03:11:12.937798+0000 mon.c (mon.2) 121 : audit 1 from='client.? 172.21.15.145:0/3656278432' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:14.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:13 smithi189 bash[23614]: audit 2024-03-07T03:11:12.947145+0000 mon.a (mon.0) 859 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:14.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:13 smithi189 bash[23614]: audit 2024-03-07T03:11:13.306708+0000 mon.a (mon.0) 860 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:13 smithi189 bash[23614]: audit 2024-03-07T03:11:13.312635+0000 mon.a (mon.0) 861 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:13 smithi189 bash[23614]: audit 2024-03-07T03:11:13.618425+0000 mon.a (mon.0) 862 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:13 smithi189 bash[23614]: audit 2024-03-07T03:11:13.638039+0000 mon.a (mon.0) 863 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:14.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:13 smithi189 bash[23614]: audit 2024-03-07T03:11:13.642323+0000 mon.c (mon.2) 122 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:14.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:13 smithi189 bash[23614]: audit 2024-03-07T03:11:13.644059+0000 mon.c (mon.2) 123 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:14.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:13 smithi189 bash[23614]: audit 2024-03-07T03:11:13.659218+0000 mon.a (mon.0) 864 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:15.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:15 smithi145 bash[21339]: audit 2024-03-07T03:11:13.940260+0000 mon.a (mon.0) 865 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:15.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:15 smithi145 bash[21339]: cluster 2024-03-07T03:11:13.940614+0000 mon.a (mon.0) 866 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2024-03-07T03:11:15.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:15 smithi145 bash[30906]: audit 2024-03-07T03:11:13.940260+0000 mon.a (mon.0) 865 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:15.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:15 smithi145 bash[30906]: cluster 2024-03-07T03:11:13.940614+0000 mon.a (mon.0) 866 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2024-03-07T03:11:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:14 smithi189 bash[23614]: audit 2024-03-07T03:11:13.940260+0000 mon.a (mon.0) 865 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:14 smithi189 bash[23614]: cluster 2024-03-07T03:11:13.940614+0000 mon.a (mon.0) 866 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2024-03-07T03:11:15.991 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:109: Skipped 2024-03-07T03:11:15.991 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:15.991 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.AioDiscardEvent (2032 ms) 2024-03-07T03:11:15.991 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.AioWriteEvent 2024-03-07T03:11:16.010 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:200: Skipped 2024-03-07T03:11:16.010 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.010 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.AioWriteEvent (20 ms) 2024-03-07T03:11:16.011 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.AioFlushEvent 2024-03-07T03:11:16.024 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:261: Skipped 2024-03-07T03:11:16.024 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.024 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.AioFlushEvent (13 ms) 2024-03-07T03:11:16.024 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.SnapCreate 2024-03-07T03:11:16.039 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:307: Skipped 2024-03-07T03:11:16.039 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.039 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.SnapCreate (15 ms) 2024-03-07T03:11:16.039 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.SnapProtect 2024-03-07T03:11:16.054 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:348: Skipped 2024-03-07T03:11:16.054 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.054 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.SnapProtect (15 ms) 2024-03-07T03:11:16.054 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.SnapUnprotect 2024-03-07T03:11:16.070 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:394: Skipped 2024-03-07T03:11:16.070 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.070 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.SnapUnprotect (16 ms) 2024-03-07T03:11:16.070 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.SnapRename 2024-03-07T03:11:16.084 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:450: Skipped 2024-03-07T03:11:16.085 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.085 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.SnapRename (15 ms) 2024-03-07T03:11:16.085 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.SnapRollback 2024-03-07T03:11:16.098 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:500: Skipped 2024-03-07T03:11:16.098 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.098 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.SnapRollback (14 ms) 2024-03-07T03:11:16.098 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.SnapRemove 2024-03-07T03:11:16.112 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:541: Skipped 2024-03-07T03:11:16.112 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.112 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.SnapRemove (14 ms) 2024-03-07T03:11:16.113 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.Rename 2024-03-07T03:11:16.126 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:590: Skipped 2024-03-07T03:11:16.127 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.127 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.Rename (14 ms) 2024-03-07T03:11:16.127 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.Resize 2024-03-07T03:11:16.141 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:624: Skipped 2024-03-07T03:11:16.141 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.141 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.Resize (15 ms) 2024-03-07T03:11:16.141 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.Flatten 2024-03-07T03:11:16.154 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:657: Skipped 2024-03-07T03:11:16.154 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.Flatten (12 ms) 2024-03-07T03:11:16.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.UpdateFeatures 2024-03-07T03:11:16.167 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:704: Skipped 2024-03-07T03:11:16.167 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.167 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.UpdateFeatures (13 ms) 2024-03-07T03:11:16.167 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.MetadataSet 2024-03-07T03:11:16.179 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:741: Skipped 2024-03-07T03:11:16.179 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.180 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.MetadataSet (13 ms) 2024-03-07T03:11:16.180 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.MetadataRemove 2024-03-07T03:11:16.193 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:781: Skipped 2024-03-07T03:11:16.193 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.193 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.MetadataRemove (13 ms) 2024-03-07T03:11:16.193 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalReplay.ObjectPosition 2024-03-07T03:11:16.207 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Replay.cc:824: Skipped 2024-03-07T03:11:16.207 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:16.207 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.ObjectPosition (14 ms) 2024-03-07T03:11:16.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:15 smithi145 bash[21339]: cluster 2024-03-07T03:11:14.458953+0000 mgr.y (mgr.24302) 198 : cluster 0 pgmap v178: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:16.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:15 smithi145 bash[21339]: cluster 2024-03-07T03:11:14.967823+0000 mon.a (mon.0) 867 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2024-03-07T03:11:16.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:15 smithi145 bash[30906]: cluster 2024-03-07T03:11:14.458953+0000 mgr.y (mgr.24302) 198 : cluster 0 pgmap v178: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:16.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:15 smithi145 bash[30906]: cluster 2024-03-07T03:11:14.967823+0000 mon.a (mon.0) 867 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2024-03-07T03:11:16.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:15 smithi189 bash[23614]: cluster 2024-03-07T03:11:14.458953+0000 mgr.y (mgr.24302) 198 : cluster 0 pgmap v178: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:16.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:15 smithi189 bash[23614]: cluster 2024-03-07T03:11:14.967823+0000 mon.a (mon.0) 867 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2024-03-07T03:11:16.987 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 16 tests from TestJournalReplay (2248 ms total) 2024-03-07T03:11:16.988 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:11:16.988 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 4 tests from TestJournalEntries 2024-03-07T03:11:17.195 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-03-07T03:11:17.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:16 smithi145 bash[21339]: cluster 2024-03-07T03:11:15.976157+0000 mon.a (mon.0) 868 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2024-03-07T03:11:17.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:16 smithi145 bash[30906]: cluster 2024-03-07T03:11:15.976157+0000 mon.a (mon.0) 868 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2024-03-07T03:11:17.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:16 smithi189 bash[23614]: cluster 2024-03-07T03:11:15.976157+0000 mon.a (mon.0) 868 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2024-03-07T03:11:18.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:18 smithi145 bash[21339]: cluster 2024-03-07T03:11:16.459351+0000 mgr.y (mgr.24302) 199 : cluster 0 pgmap v181: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:18.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:18 smithi145 bash[21339]: cluster 2024-03-07T03:11:16.992794+0000 mon.a (mon.0) 869 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2024-03-07T03:11:18.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:18 smithi145 bash[21339]: audit 2024-03-07T03:11:17.453308+0000 mon.c (mon.2) 124 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:18.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:18 smithi145 bash[30906]: cluster 2024-03-07T03:11:16.459351+0000 mgr.y (mgr.24302) 199 : cluster 0 pgmap v181: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:18.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:18 smithi145 bash[30906]: cluster 2024-03-07T03:11:16.992794+0000 mon.a (mon.0) 869 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2024-03-07T03:11:18.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:18 smithi145 bash[30906]: audit 2024-03-07T03:11:17.453308+0000 mon.c (mon.2) 124 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:18.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:18 smithi189 bash[23614]: cluster 2024-03-07T03:11:16.459351+0000 mgr.y (mgr.24302) 199 : cluster 0 pgmap v181: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:18.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:18 smithi189 bash[23614]: cluster 2024-03-07T03:11:16.992794+0000 mon.a (mon.0) 869 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2024-03-07T03:11:18.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:18 smithi189 bash[23614]: audit 2024-03-07T03:11:17.453308+0000 mon.c (mon.2) 124 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:18.994 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalEntries.AioWrite 2024-03-07T03:11:19.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:19 smithi145 bash[21339]: cluster 2024-03-07T03:11:17.993130+0000 mon.a (mon.0) 870 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2024-03-07T03:11:19.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:19 smithi145 bash[21339]: audit 2024-03-07T03:11:18.007448+0000 mon.a (mon.0) 871 : audit 1 from='client.? 172.21.15.145:0/1936562817' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:19.315 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:19 smithi145 bash[21339]: audit 2024-03-07T03:11:19.000549+0000 mon.a (mon.0) 872 : audit 1 from='client.? 172.21.15.145:0/1936562817' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:19.315 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:19 smithi145 bash[21339]: cluster 2024-03-07T03:11:19.001822+0000 mon.a (mon.0) 873 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2024-03-07T03:11:19.315 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:19 smithi145 bash[30906]: cluster 2024-03-07T03:11:17.993130+0000 mon.a (mon.0) 870 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2024-03-07T03:11:19.315 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:19 smithi145 bash[30906]: audit 2024-03-07T03:11:18.007448+0000 mon.a (mon.0) 871 : audit 1 from='client.? 172.21.15.145:0/1936562817' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:19.315 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:19 smithi145 bash[30906]: audit 2024-03-07T03:11:19.000549+0000 mon.a (mon.0) 872 : audit 1 from='client.? 172.21.15.145:0/1936562817' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:19.315 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:19 smithi145 bash[30906]: cluster 2024-03-07T03:11:19.001822+0000 mon.a (mon.0) 873 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2024-03-07T03:11:19.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:19 smithi189 bash[23614]: cluster 2024-03-07T03:11:17.993130+0000 mon.a (mon.0) 870 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2024-03-07T03:11:19.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:19 smithi189 bash[23614]: audit 2024-03-07T03:11:18.007448+0000 mon.a (mon.0) 871 : audit 1 from='client.? 172.21.15.145:0/1936562817' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:19.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:19 smithi189 bash[23614]: audit 2024-03-07T03:11:19.000549+0000 mon.a (mon.0) 872 : audit 1 from='client.? 172.21.15.145:0/1936562817' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:19.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:19 smithi189 bash[23614]: cluster 2024-03-07T03:11:19.001822+0000 mon.a (mon.0) 873 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2024-03-07T03:11:20.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:20 smithi145 bash[21339]: cluster 2024-03-07T03:11:18.460077+0000 mgr.y (mgr.24302) 200 : cluster 0 pgmap v184: 105 pgs: 2 creating+peering, 30 unknown, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:20.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:20 smithi145 bash[21339]: cluster 2024-03-07T03:11:19.398565+0000 mon.a (mon.0) 874 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:20.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:20 smithi145 bash[21339]: cluster 2024-03-07T03:11:19.994056+0000 mon.a (mon.0) 875 : cluster 0 osdmap e85: 8 total, 8 up, 8 in 2024-03-07T03:11:20.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:20 smithi145 bash[30906]: cluster 2024-03-07T03:11:18.460077+0000 mgr.y (mgr.24302) 200 : cluster 0 pgmap v184: 105 pgs: 2 creating+peering, 30 unknown, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:20.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:20 smithi145 bash[30906]: cluster 2024-03-07T03:11:19.398565+0000 mon.a (mon.0) 874 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:20.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:20 smithi145 bash[30906]: cluster 2024-03-07T03:11:19.994056+0000 mon.a (mon.0) 875 : cluster 0 osdmap e85: 8 total, 8 up, 8 in 2024-03-07T03:11:20.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:20 smithi189 bash[23614]: cluster 2024-03-07T03:11:18.460077+0000 mgr.y (mgr.24302) 200 : cluster 0 pgmap v184: 105 pgs: 2 creating+peering, 30 unknown, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:20.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:20 smithi189 bash[23614]: cluster 2024-03-07T03:11:19.398565+0000 mon.a (mon.0) 874 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:20.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:20 smithi189 bash[23614]: cluster 2024-03-07T03:11:19.994056+0000 mon.a (mon.0) 875 : cluster 0 osdmap e85: 8 total, 8 up, 8 in 2024-03-07T03:11:21.027 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Entries.cc:118: Skipped 2024-03-07T03:11:21.027 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:21.027 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalEntries.AioWrite (2033 ms) 2024-03-07T03:11:21.027 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalEntries.AioDiscard 2024-03-07T03:11:21.049 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Entries.cc:163: Skipped 2024-03-07T03:11:21.049 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:21.049 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalEntries.AioDiscard (22 ms) 2024-03-07T03:11:21.049 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalEntries.AioDiscardWithPrune 2024-03-07T03:11:21.067 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Entries.cc:200: Skipped 2024-03-07T03:11:21.068 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:21.068 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalEntries.AioDiscardWithPrune (19 ms) 2024-03-07T03:11:21.068 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestJournalEntries.AioFlush 2024-03-07T03:11:21.085 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/journal/test_Entries.cc:263: Skipped 2024-03-07T03:11:21.085 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:21.086 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalEntries.AioFlush (18 ms) 2024-03-07T03:11:21.933 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:11:22.021 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 4 tests from TestJournalEntries (2092 ms total) 2024-03-07T03:11:22.022 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:11:22.022 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 2 tests from TestTrash 2024-03-07T03:11:22.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:21 smithi145 bash[30906]: cluster 2024-03-07T03:11:20.460791+0000 mgr.y (mgr.24302) 201 : cluster 0 pgmap v187: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:22.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:21 smithi145 bash[30906]: cluster 2024-03-07T03:11:20.999550+0000 mon.a (mon.0) 876 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2024-03-07T03:11:22.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:21 smithi145 bash[30906]: cluster 2024-03-07T03:11:21.018645+0000 mon.a (mon.0) 877 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:22.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:21 smithi145 bash[30906]: cluster 2024-03-07T03:11:21.018726+0000 mon.a (mon.0) 878 : cluster 1 Cluster is now healthy 2024-03-07T03:11:22.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:21 smithi145 bash[21339]: cluster 2024-03-07T03:11:20.460791+0000 mgr.y (mgr.24302) 201 : cluster 0 pgmap v187: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:22.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:21 smithi145 bash[21339]: cluster 2024-03-07T03:11:20.999550+0000 mon.a (mon.0) 876 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2024-03-07T03:11:22.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:21 smithi145 bash[21339]: cluster 2024-03-07T03:11:21.018645+0000 mon.a (mon.0) 877 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:22.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:21 smithi145 bash[21339]: cluster 2024-03-07T03:11:21.018726+0000 mon.a (mon.0) 878 : cluster 1 Cluster is now healthy 2024-03-07T03:11:22.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:21 smithi189 bash[23614]: cluster 2024-03-07T03:11:20.460791+0000 mgr.y (mgr.24302) 201 : cluster 0 pgmap v187: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:22.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:21 smithi189 bash[23614]: cluster 2024-03-07T03:11:20.999550+0000 mon.a (mon.0) 876 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2024-03-07T03:11:22.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:21 smithi189 bash[23614]: cluster 2024-03-07T03:11:21.018645+0000 mon.a (mon.0) 877 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:22.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:21 smithi189 bash[23614]: cluster 2024-03-07T03:11:21.018726+0000 mon.a (mon.0) 878 : cluster 1 Cluster is now healthy 2024-03-07T03:11:22.811 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:11:22 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:11:22] "GET /metrics HTTP/1.1" 200 33763 "" "Prometheus/2.43.0" 2024-03-07T03:11:23.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:23 smithi145 bash[21339]: cluster 2024-03-07T03:11:22.028282+0000 mon.a (mon.0) 879 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2024-03-07T03:11:23.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:23 smithi145 bash[30906]: cluster 2024-03-07T03:11:22.028282+0000 mon.a (mon.0) 879 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2024-03-07T03:11:23.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:23 smithi189 bash[23614]: cluster 2024-03-07T03:11:22.028282+0000 mon.a (mon.0) 879 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2024-03-07T03:11:24.038 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestTrash.UserRemovingSource 2024-03-07T03:11:24.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:24 smithi145 bash[21339]: cluster 2024-03-07T03:11:22.461398+0000 mgr.y (mgr.24302) 202 : cluster 0 pgmap v190: 73 pgs: 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:24.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:24 smithi145 bash[21339]: cluster 2024-03-07T03:11:23.044553+0000 mon.a (mon.0) 880 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2024-03-07T03:11:24.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:24 smithi145 bash[21339]: audit 2024-03-07T03:11:23.048638+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.145:0/2092203371' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:24.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:24 smithi145 bash[21339]: audit 2024-03-07T03:11:23.049669+0000 mon.a (mon.0) 881 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:24.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:24 smithi145 bash[30906]: cluster 2024-03-07T03:11:22.461398+0000 mgr.y (mgr.24302) 202 : cluster 0 pgmap v190: 73 pgs: 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:24.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:24 smithi145 bash[30906]: cluster 2024-03-07T03:11:23.044553+0000 mon.a (mon.0) 880 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2024-03-07T03:11:24.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:24 smithi145 bash[30906]: audit 2024-03-07T03:11:23.048638+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.145:0/2092203371' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:24.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:24 smithi145 bash[30906]: audit 2024-03-07T03:11:23.049669+0000 mon.a (mon.0) 881 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:24.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:24 smithi189 bash[23614]: cluster 2024-03-07T03:11:22.461398+0000 mgr.y (mgr.24302) 202 : cluster 0 pgmap v190: 73 pgs: 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:24.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:24 smithi189 bash[23614]: cluster 2024-03-07T03:11:23.044553+0000 mon.a (mon.0) 880 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2024-03-07T03:11:24.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:24 smithi189 bash[23614]: audit 2024-03-07T03:11:23.048638+0000 mon.c (mon.2) 125 : audit 1 from='client.? 172.21.15.145:0/2092203371' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:24.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:24 smithi189 bash[23614]: audit 2024-03-07T03:11:23.049669+0000 mon.a (mon.0) 881 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:25.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:25 smithi145 bash[30906]: audit 2024-03-07T03:11:24.046021+0000 mon.a (mon.0) 882 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:25.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:25 smithi145 bash[30906]: cluster 2024-03-07T03:11:24.046613+0000 mon.a (mon.0) 883 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2024-03-07T03:11:25.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:25 smithi145 bash[21339]: audit 2024-03-07T03:11:24.046021+0000 mon.a (mon.0) 882 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:25.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:25 smithi145 bash[21339]: cluster 2024-03-07T03:11:24.046613+0000 mon.a (mon.0) 883 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2024-03-07T03:11:25.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:25 smithi189 bash[23614]: audit 2024-03-07T03:11:24.046021+0000 mon.a (mon.0) 882 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:25.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:25 smithi189 bash[23614]: cluster 2024-03-07T03:11:24.046613+0000 mon.a (mon.0) 883 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2024-03-07T03:11:25.576 INFO:teuthology.orchestra.run.smithi145.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:11:26.217 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestTrash.UserRemovingSource (2179 ms) 2024-03-07T03:11:26.217 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestTrash.RestoreMirroringSource 2024-03-07T03:11:26.281 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestTrash.RestoreMirroringSource (64 ms) 2024-03-07T03:11:26.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[30906]: cluster 2024-03-07T03:11:24.462023+0000 mgr.y (mgr.24302) 203 : cluster 0 pgmap v193: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:26.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[30906]: cluster 2024-03-07T03:11:25.064242+0000 mon.a (mon.0) 884 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2024-03-07T03:11:26.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[30906]: audit 2024-03-07T03:11:25.584112+0000 mon.a (mon.0) 885 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:26.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[30906]: audit 2024-03-07T03:11:25.598675+0000 mon.c (mon.2) 126 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:26.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[30906]: audit 2024-03-07T03:11:25.601752+0000 mon.c (mon.2) 127 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:26.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[30906]: audit 2024-03-07T03:11:25.603259+0000 mon.c (mon.2) 128 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:26.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[30906]: audit 2024-03-07T03:11:25.613006+0000 mon.a (mon.0) 886 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:26.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[21339]: cluster 2024-03-07T03:11:24.462023+0000 mgr.y (mgr.24302) 203 : cluster 0 pgmap v193: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:26.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[21339]: cluster 2024-03-07T03:11:25.064242+0000 mon.a (mon.0) 884 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2024-03-07T03:11:26.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[21339]: audit 2024-03-07T03:11:25.584112+0000 mon.a (mon.0) 885 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:26.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[21339]: audit 2024-03-07T03:11:25.598675+0000 mon.c (mon.2) 126 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:26.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[21339]: audit 2024-03-07T03:11:25.601752+0000 mon.c (mon.2) 127 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:26.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[21339]: audit 2024-03-07T03:11:25.603259+0000 mon.c (mon.2) 128 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:26.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:26 smithi145 bash[21339]: audit 2024-03-07T03:11:25.613006+0000 mon.a (mon.0) 886 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:26.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:26 smithi189 bash[23614]: cluster 2024-03-07T03:11:24.462023+0000 mgr.y (mgr.24302) 203 : cluster 0 pgmap v193: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:26.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:26 smithi189 bash[23614]: cluster 2024-03-07T03:11:25.064242+0000 mon.a (mon.0) 884 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2024-03-07T03:11:26.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:26 smithi189 bash[23614]: audit 2024-03-07T03:11:25.584112+0000 mon.a (mon.0) 885 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:26.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:26 smithi189 bash[23614]: audit 2024-03-07T03:11:25.598675+0000 mon.c (mon.2) 126 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:26.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:26 smithi189 bash[23614]: audit 2024-03-07T03:11:25.601752+0000 mon.c (mon.2) 127 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:26.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:26 smithi189 bash[23614]: audit 2024-03-07T03:11:25.603259+0000 mon.c (mon.2) 128 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:26.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:26 smithi189 bash[23614]: audit 2024-03-07T03:11:25.613006+0000 mon.a (mon.0) 886 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:26.479 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch ps' 2024-03-07T03:11:27.103 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 2 tests from TestTrash (2243 ms total) 2024-03-07T03:11:27.103 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:11:27.103 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 1 test from TestOperations 2024-03-07T03:11:27.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:27 smithi145 bash[21339]: audit 2024-03-07T03:11:25.574263+0000 mgr.y (mgr.24302) 204 : audit 0 from='client.14805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:11:27.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:27 smithi145 bash[21339]: cephadm 2024-03-07T03:11:25.575794+0000 mgr.y (mgr.24302) 205 : cephadm 1 Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:11:27.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:27 smithi145 bash[21339]: cephadm 2024-03-07T03:11:25.687513+0000 mgr.y (mgr.24302) 206 : cephadm 1 Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:11:27.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:27 smithi145 bash[21339]: cluster 2024-03-07T03:11:26.077031+0000 mon.a (mon.0) 887 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2024-03-07T03:11:27.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:27 smithi145 bash[30906]: audit 2024-03-07T03:11:25.574263+0000 mgr.y (mgr.24302) 204 : audit 0 from='client.14805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:11:27.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:27 smithi145 bash[30906]: cephadm 2024-03-07T03:11:25.575794+0000 mgr.y (mgr.24302) 205 : cephadm 1 Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:11:27.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:27 smithi145 bash[30906]: cephadm 2024-03-07T03:11:25.687513+0000 mgr.y (mgr.24302) 206 : cephadm 1 Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:11:27.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:27 smithi145 bash[30906]: cluster 2024-03-07T03:11:26.077031+0000 mon.a (mon.0) 887 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2024-03-07T03:11:27.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:27 smithi189 bash[23614]: audit 2024-03-07T03:11:25.574263+0000 mgr.y (mgr.24302) 204 : audit 0 from='client.14805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:11:27.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:27 smithi189 bash[23614]: cephadm 2024-03-07T03:11:25.575794+0000 mgr.y (mgr.24302) 205 : cephadm 1 Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:11:27.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:27 smithi189 bash[23614]: cephadm 2024-03-07T03:11:25.687513+0000 mgr.y (mgr.24302) 206 : cephadm 1 Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:11:27.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:27 smithi189 bash[23614]: cluster 2024-03-07T03:11:26.077031+0000 mon.a (mon.0) 887 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2024-03-07T03:11:28.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:28 smithi145 bash[21339]: cluster 2024-03-07T03:11:26.462500+0000 mgr.y (mgr.24302) 207 : cluster 0 pgmap v196: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:28.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:28 smithi145 bash[21339]: cluster 2024-03-07T03:11:27.103543+0000 mon.a (mon.0) 888 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2024-03-07T03:11:28.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:28 smithi145 bash[30906]: cluster 2024-03-07T03:11:26.462500+0000 mgr.y (mgr.24302) 207 : cluster 0 pgmap v196: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:28.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:28 smithi145 bash[30906]: cluster 2024-03-07T03:11:27.103543+0000 mon.a (mon.0) 888 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2024-03-07T03:11:28.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:28 smithi189 bash[23614]: cluster 2024-03-07T03:11:26.462500+0000 mgr.y (mgr.24302) 207 : cluster 0 pgmap v196: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:28.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:28 smithi189 bash[23614]: cluster 2024-03-07T03:11:27.103543+0000 mon.a (mon.0) 888 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2024-03-07T03:11:29.107 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestOperations.DisableJournalingCorrupt 2024-03-07T03:11:29.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:29 smithi145 bash[30906]: cluster 2024-03-07T03:11:28.104403+0000 mon.a (mon.0) 889 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2024-03-07T03:11:29.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:29 smithi145 bash[30906]: audit 2024-03-07T03:11:28.112541+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.145:0/2328780863' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:29.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:29 smithi145 bash[30906]: audit 2024-03-07T03:11:28.124592+0000 mon.a (mon.0) 890 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:29.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:29 smithi145 bash[30906]: audit 2024-03-07T03:11:29.110446+0000 mon.a (mon.0) 891 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:29.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:29 smithi145 bash[30906]: cluster 2024-03-07T03:11:29.110870+0000 mon.a (mon.0) 892 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2024-03-07T03:11:29.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:29 smithi145 bash[21339]: cluster 2024-03-07T03:11:28.104403+0000 mon.a (mon.0) 889 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2024-03-07T03:11:29.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:29 smithi145 bash[21339]: audit 2024-03-07T03:11:28.112541+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.145:0/2328780863' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:29.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:29 smithi145 bash[21339]: audit 2024-03-07T03:11:28.124592+0000 mon.a (mon.0) 890 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:29.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:29 smithi145 bash[21339]: audit 2024-03-07T03:11:29.110446+0000 mon.a (mon.0) 891 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:29.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:29 smithi145 bash[21339]: cluster 2024-03-07T03:11:29.110870+0000 mon.a (mon.0) 892 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2024-03-07T03:11:29.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:29 smithi189 bash[23614]: cluster 2024-03-07T03:11:28.104403+0000 mon.a (mon.0) 889 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2024-03-07T03:11:29.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:29 smithi189 bash[23614]: audit 2024-03-07T03:11:28.112541+0000 mon.c (mon.2) 129 : audit 1 from='client.? 172.21.15.145:0/2328780863' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:29.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:29 smithi189 bash[23614]: audit 2024-03-07T03:11:28.124592+0000 mon.a (mon.0) 890 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:29.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:29 smithi189 bash[23614]: audit 2024-03-07T03:11:29.110446+0000 mon.a (mon.0) 891 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:29.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:29 smithi189 bash[23614]: cluster 2024-03-07T03:11:29.110870+0000 mon.a (mon.0) 892 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2024-03-07T03:11:30.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:30 smithi145 bash[21339]: cluster 2024-03-07T03:11:28.463215+0000 mgr.y (mgr.24302) 208 : cluster 0 pgmap v199: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:30.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:30 smithi145 bash[21339]: cluster 2024-03-07T03:11:29.401535+0000 mon.a (mon.0) 893 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:30.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:30 smithi145 bash[30906]: cluster 2024-03-07T03:11:28.463215+0000 mgr.y (mgr.24302) 208 : cluster 0 pgmap v199: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:30.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:30 smithi145 bash[30906]: cluster 2024-03-07T03:11:29.401535+0000 mon.a (mon.0) 893 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:30.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:30 smithi189 bash[23614]: cluster 2024-03-07T03:11:28.463215+0000 mgr.y (mgr.24302) 208 : cluster 0 pgmap v199: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:30.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:30 smithi189 bash[23614]: cluster 2024-03-07T03:11:29.401535+0000 mon.a (mon.0) 893 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:31.174 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_Operations.cc:18: Skipped 2024-03-07T03:11:31.174 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:31.174 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestOperations.DisableJournalingCorrupt (2066 ms) 2024-03-07T03:11:31.196 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:11:31.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:31 smithi145 bash[21339]: cluster 2024-03-07T03:11:30.131544+0000 mon.a (mon.0) 894 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2024-03-07T03:11:31.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:31 smithi145 bash[30906]: cluster 2024-03-07T03:11:30.131544+0000 mon.a (mon.0) 894 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2024-03-07T03:11:31.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:31 smithi189 bash[23614]: cluster 2024-03-07T03:11:30.131544+0000 mon.a (mon.0) 894 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2024-03-07T03:11:32.159 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 1 test from TestOperations (2066 ms total) 2024-03-07T03:11:32.159 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:11:32.159 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 4 tests from TestObjectMap 2024-03-07T03:11:32.561 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:11:32 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:11:32] "GET /metrics HTTP/1.1" 200 33763 "" "Prometheus/2.43.0" 2024-03-07T03:11:32.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:32 smithi145 bash[21339]: cluster 2024-03-07T03:11:30.463919+0000 mgr.y (mgr.24302) 209 : cluster 0 pgmap v202: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:32.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:32 smithi145 bash[21339]: cluster 2024-03-07T03:11:31.133941+0000 mon.a (mon.0) 895 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:32.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:32 smithi145 bash[21339]: cluster 2024-03-07T03:11:31.134002+0000 mon.a (mon.0) 896 : cluster 1 Cluster is now healthy 2024-03-07T03:11:32.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:32 smithi145 bash[21339]: cluster 2024-03-07T03:11:31.150583+0000 mon.a (mon.0) 897 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2024-03-07T03:11:32.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:32 smithi145 bash[30906]: cluster 2024-03-07T03:11:30.463919+0000 mgr.y (mgr.24302) 209 : cluster 0 pgmap v202: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:32.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:32 smithi145 bash[30906]: cluster 2024-03-07T03:11:31.133941+0000 mon.a (mon.0) 895 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:32.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:32 smithi145 bash[30906]: cluster 2024-03-07T03:11:31.134002+0000 mon.a (mon.0) 896 : cluster 1 Cluster is now healthy 2024-03-07T03:11:32.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:32 smithi145 bash[30906]: cluster 2024-03-07T03:11:31.150583+0000 mon.a (mon.0) 897 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2024-03-07T03:11:32.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:32 smithi189 bash[23614]: cluster 2024-03-07T03:11:30.463919+0000 mgr.y (mgr.24302) 209 : cluster 0 pgmap v202: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:32.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:32 smithi189 bash[23614]: cluster 2024-03-07T03:11:31.133941+0000 mon.a (mon.0) 895 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:32.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:32 smithi189 bash[23614]: cluster 2024-03-07T03:11:31.134002+0000 mon.a (mon.0) 896 : cluster 1 Cluster is now healthy 2024-03-07T03:11:32.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:32 smithi189 bash[23614]: cluster 2024-03-07T03:11:31.150583+0000 mon.a (mon.0) 897 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2024-03-07T03:11:33.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:33 smithi145 bash[21339]: cluster 2024-03-07T03:11:32.160185+0000 mon.a (mon.0) 898 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2024-03-07T03:11:33.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:33 smithi145 bash[21339]: audit 2024-03-07T03:11:32.452712+0000 mon.c (mon.2) 130 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:33.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:33 smithi145 bash[30906]: cluster 2024-03-07T03:11:32.160185+0000 mon.a (mon.0) 898 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2024-03-07T03:11:33.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:33 smithi145 bash[30906]: audit 2024-03-07T03:11:32.452712+0000 mon.c (mon.2) 130 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:33.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:33 smithi189 bash[23614]: cluster 2024-03-07T03:11:32.160185+0000 mon.a (mon.0) 898 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2024-03-07T03:11:33.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:33 smithi189 bash[23614]: audit 2024-03-07T03:11:32.452712+0000 mon.c (mon.2) 130 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:34.170 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenCorrupt 2024-03-07T03:11:34.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:34 smithi145 bash[21339]: cluster 2024-03-07T03:11:32.464653+0000 mgr.y (mgr.24302) 210 : cluster 0 pgmap v205: 73 pgs: 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:34.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:34 smithi145 bash[21339]: cluster 2024-03-07T03:11:33.179087+0000 mon.a (mon.0) 899 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2024-03-07T03:11:34.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:34 smithi145 bash[21339]: audit 2024-03-07T03:11:33.192558+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.145:0/2495177538' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:34.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:34 smithi145 bash[21339]: audit 2024-03-07T03:11:33.193482+0000 mon.a (mon.0) 900 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:34.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:34 smithi145 bash[30906]: cluster 2024-03-07T03:11:32.464653+0000 mgr.y (mgr.24302) 210 : cluster 0 pgmap v205: 73 pgs: 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:34.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:34 smithi145 bash[30906]: cluster 2024-03-07T03:11:33.179087+0000 mon.a (mon.0) 899 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2024-03-07T03:11:34.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:34 smithi145 bash[30906]: audit 2024-03-07T03:11:33.192558+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.145:0/2495177538' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:34.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:34 smithi145 bash[30906]: audit 2024-03-07T03:11:33.193482+0000 mon.a (mon.0) 900 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:34.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:34 smithi189 bash[23614]: cluster 2024-03-07T03:11:32.464653+0000 mgr.y (mgr.24302) 210 : cluster 0 pgmap v205: 73 pgs: 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:34.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:34 smithi189 bash[23614]: cluster 2024-03-07T03:11:33.179087+0000 mon.a (mon.0) 899 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2024-03-07T03:11:34.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:34 smithi189 bash[23614]: audit 2024-03-07T03:11:33.192558+0000 mon.c (mon.2) 131 : audit 1 from='client.? 172.21.15.145:0/2495177538' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:34.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:34 smithi189 bash[23614]: audit 2024-03-07T03:11:33.193482+0000 mon.a (mon.0) 900 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:35.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:35 smithi145 bash[21339]: audit 2024-03-07T03:11:34.177160+0000 mon.a (mon.0) 901 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:35.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:35 smithi145 bash[21339]: cluster 2024-03-07T03:11:34.177407+0000 mon.a (mon.0) 902 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2024-03-07T03:11:35.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:35 smithi145 bash[30906]: audit 2024-03-07T03:11:34.177160+0000 mon.a (mon.0) 901 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:35.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:35 smithi145 bash[30906]: cluster 2024-03-07T03:11:34.177407+0000 mon.a (mon.0) 902 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2024-03-07T03:11:35.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:35 smithi189 bash[23614]: audit 2024-03-07T03:11:34.177160+0000 mon.a (mon.0) 901 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:35.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:35 smithi189 bash[23614]: cluster 2024-03-07T03:11:34.177407+0000 mon.a (mon.0) 902 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2024-03-07T03:11:36.252 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenCorrupt (2082 ms) 2024-03-07T03:11:36.252 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenTooSmall 2024-03-07T03:11:36.297 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenTooSmall (45 ms) 2024-03-07T03:11:36.297 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestObjectMap.InvalidateFlagOnDisk 2024-03-07T03:11:36.365 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestObjectMap.InvalidateFlagOnDisk (68 ms) 2024-03-07T03:11:36.365 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall 2024-03-07T03:11:36.418 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall (53 ms) 2024-03-07T03:11:36.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:36 smithi145 bash[21339]: cluster 2024-03-07T03:11:34.465367+0000 mgr.y (mgr.24302) 211 : cluster 0 pgmap v208: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:36.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:36 smithi145 bash[21339]: cluster 2024-03-07T03:11:35.189724+0000 mon.a (mon.0) 903 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2024-03-07T03:11:36.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:36 smithi145 bash[30906]: cluster 2024-03-07T03:11:34.465367+0000 mgr.y (mgr.24302) 211 : cluster 0 pgmap v208: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:36.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:36 smithi145 bash[30906]: cluster 2024-03-07T03:11:35.189724+0000 mon.a (mon.0) 903 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2024-03-07T03:11:36.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:36 smithi189 bash[23614]: cluster 2024-03-07T03:11:34.465367+0000 mgr.y (mgr.24302) 211 : cluster 0 pgmap v208: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:36.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:36 smithi189 bash[23614]: cluster 2024-03-07T03:11:35.189724+0000 mon.a (mon.0) 903 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2024-03-07T03:11:36.612 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-07T03:11:36.612 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (4m) 23s ago 5m 15.0M - 0.25.0 c8568f914cd2 cfa1e64a6007 2024-03-07T03:11:36.612 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi189 *:3000 running (4m) 23s ago 4m 78.1M - 9.4.7 954c08fa6188 89badf11bd11 2024-03-07T03:11:36.612 INFO:teuthology.orchestra.run.smithi145.stdout:mds.foo.smithi145.zmgvgw smithi145 running (2m) 23s ago 2m 12.0M - 18.2.1-718-gae75fc9d 0164bb8a7c67 4ded79fef80e 2024-03-07T03:11:36.612 INFO:teuthology.orchestra.run.smithi145.stdout:mds.foo.smithi189.zkinbe smithi189 running (2m) 23s ago 2m 13.6M - 18.2.1-718-gae75fc9d 0164bb8a7c67 6589deb5c181 2024-03-07T03:11:36.612 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi189 *:8443,8765 running (13m) 23s ago 13m 436M - 18.2.1-718-gae75fc9d 0164bb8a7c67 b61d3ef7d2d3 2024-03-07T03:11:36.612 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:9283,8765 running (15m) 23s ago 15m 491M - 18.2.1-718-gae75fc9d 0164bb8a7c67 70ce3d1d21be 2024-03-07T03:11:36.612 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (15m) 23s ago 15m 49.4M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 ba6b29c5069f 2024-03-07T03:11:36.612 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi189 running (13m) 23s ago 13m 39.4M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 5746e42ef7c9 2024-03-07T03:11:36.613 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (13m) 23s ago 13m 39.9M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 eb9564943cff 2024-03-07T03:11:36.613 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (5m) 23s ago 5m 8143k - 1.5.0 0da6a335fe13 895e660f39b6 2024-03-07T03:11:36.613 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi189 *:9100 running (5m) 23s ago 5m 8720k - 1.5.0 0da6a335fe13 ab7ffb60c1e8 2024-03-07T03:11:36.613 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (12m) 23s ago 12m 46.5M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 4485c22f1548 2024-03-07T03:11:36.613 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (11m) 23s ago 11m 46.8M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 f21d0bf7ebc8 2024-03-07T03:11:36.613 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (10m) 23s ago 10m 43.0M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 9e8c1bf873ba 2024-03-07T03:11:36.613 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (9m) 23s ago 9m 43.2M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 c209b1a43303 2024-03-07T03:11:36.613 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi189 running (9m) 23s ago 9m 44.7M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 8845b4f5216a 2024-03-07T03:11:36.613 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi189 running (8m) 23s ago 8m 43.9M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 44e5411eb07c 2024-03-07T03:11:36.613 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi189 running (7m) 23s ago 7m 42.3M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 c2aeea0466c7 2024-03-07T03:11:36.613 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi189 running (6m) 23s ago 6m 43.4M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 bf101bfb6800 2024-03-07T03:11:36.613 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi189 *:9095 running (4m) 23s ago 5m 41.3M - 2.43.0 a07b618ecd1d adc2e6971315 2024-03-07T03:11:37.225 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 4 tests from TestObjectMap (2249 ms total) 2024-03-07T03:11:37.225 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:11:37.225 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 2 tests from TestMirroringWatcher 2024-03-07T03:11:37.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:37 smithi145 bash[21339]: cluster 2024-03-07T03:11:36.198401+0000 mon.a (mon.0) 904 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2024-03-07T03:11:37.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:37 smithi145 bash[30906]: cluster 2024-03-07T03:11:36.198401+0000 mon.a (mon.0) 904 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2024-03-07T03:11:37.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:37 smithi189 bash[23614]: cluster 2024-03-07T03:11:36.198401+0000 mon.a (mon.0) 904 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2024-03-07T03:11:37.753 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'echo wait for minority of mons to upgrade' 2024-03-07T03:11:38.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:38 smithi145 bash[21339]: cluster 2024-03-07T03:11:36.465831+0000 mgr.y (mgr.24302) 212 : cluster 0 pgmap v211: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:38.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:38 smithi145 bash[21339]: audit 2024-03-07T03:11:36.603645+0000 mgr.y (mgr.24302) 213 : audit 0 from='client.24749 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:11:38.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:38 smithi145 bash[21339]: cluster 2024-03-07T03:11:37.228246+0000 mon.a (mon.0) 905 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2024-03-07T03:11:38.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:38 smithi145 bash[30906]: cluster 2024-03-07T03:11:36.465831+0000 mgr.y (mgr.24302) 212 : cluster 0 pgmap v211: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:38.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:38 smithi145 bash[30906]: audit 2024-03-07T03:11:36.603645+0000 mgr.y (mgr.24302) 213 : audit 0 from='client.24749 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:11:38.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:38 smithi145 bash[30906]: cluster 2024-03-07T03:11:37.228246+0000 mon.a (mon.0) 905 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2024-03-07T03:11:38.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:38 smithi189 bash[23614]: cluster 2024-03-07T03:11:36.465831+0000 mgr.y (mgr.24302) 212 : cluster 0 pgmap v211: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:38.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:38 smithi189 bash[23614]: audit 2024-03-07T03:11:36.603645+0000 mgr.y (mgr.24302) 213 : audit 0 from='client.24749 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:11:38.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:38 smithi189 bash[23614]: cluster 2024-03-07T03:11:37.228246+0000 mon.a (mon.0) 905 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2024-03-07T03:11:39.244 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroringWatcher.ModeUpdated 2024-03-07T03:11:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:39 smithi145 bash[21339]: cluster 2024-03-07T03:11:38.244340+0000 mon.a (mon.0) 906 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2024-03-07T03:11:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:39 smithi145 bash[21339]: audit 2024-03-07T03:11:38.250710+0000 mon.c (mon.2) 132 : audit 1 from='client.? 172.21.15.145:0/640121639' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:39 smithi145 bash[21339]: audit 2024-03-07T03:11:38.252428+0000 mon.a (mon.0) 907 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:39.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:39 smithi145 bash[30906]: cluster 2024-03-07T03:11:38.244340+0000 mon.a (mon.0) 906 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2024-03-07T03:11:39.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:39 smithi145 bash[30906]: audit 2024-03-07T03:11:38.250710+0000 mon.c (mon.2) 132 : audit 1 from='client.? 172.21.15.145:0/640121639' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:39.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:39 smithi145 bash[30906]: audit 2024-03-07T03:11:38.252428+0000 mon.a (mon.0) 907 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:39.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:39 smithi189 bash[23614]: cluster 2024-03-07T03:11:38.244340+0000 mon.a (mon.0) 906 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2024-03-07T03:11:39.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:39 smithi189 bash[23614]: audit 2024-03-07T03:11:38.250710+0000 mon.c (mon.2) 132 : audit 1 from='client.? 172.21.15.145:0/640121639' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:39.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:39 smithi189 bash[23614]: audit 2024-03-07T03:11:38.252428+0000 mon.a (mon.0) 907 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:40.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:40 smithi145 bash[30906]: cluster 2024-03-07T03:11:38.466633+0000 mgr.y (mgr.24302) 214 : cluster 0 pgmap v214: 105 pgs: 9 creating+peering, 23 unknown, 73 active+clean; 579 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:40.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:40 smithi145 bash[30906]: audit 2024-03-07T03:11:39.245743+0000 mon.a (mon.0) 908 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:40.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:40 smithi145 bash[30906]: cluster 2024-03-07T03:11:39.246776+0000 mon.a (mon.0) 909 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2024-03-07T03:11:40.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:40 smithi145 bash[30906]: cluster 2024-03-07T03:11:39.404954+0000 mon.a (mon.0) 910 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:40.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:40 smithi145 bash[30906]: cluster 2024-03-07T03:11:40.249445+0000 mon.a (mon.0) 911 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2024-03-07T03:11:40.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:40 smithi145 bash[21339]: cluster 2024-03-07T03:11:38.466633+0000 mgr.y (mgr.24302) 214 : cluster 0 pgmap v214: 105 pgs: 9 creating+peering, 23 unknown, 73 active+clean; 579 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:40.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:40 smithi145 bash[21339]: audit 2024-03-07T03:11:39.245743+0000 mon.a (mon.0) 908 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:40.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:40 smithi145 bash[21339]: cluster 2024-03-07T03:11:39.246776+0000 mon.a (mon.0) 909 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2024-03-07T03:11:40.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:40 smithi145 bash[21339]: cluster 2024-03-07T03:11:39.404954+0000 mon.a (mon.0) 910 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:40.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:40 smithi145 bash[21339]: cluster 2024-03-07T03:11:40.249445+0000 mon.a (mon.0) 911 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2024-03-07T03:11:40.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:40 smithi189 bash[23614]: cluster 2024-03-07T03:11:38.466633+0000 mgr.y (mgr.24302) 214 : cluster 0 pgmap v214: 105 pgs: 9 creating+peering, 23 unknown, 73 active+clean; 579 KiB data, 225 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:40.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:40 smithi189 bash[23614]: audit 2024-03-07T03:11:39.245743+0000 mon.a (mon.0) 908 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:40.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:40 smithi189 bash[23614]: cluster 2024-03-07T03:11:39.246776+0000 mon.a (mon.0) 909 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2024-03-07T03:11:40.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:40 smithi189 bash[23614]: cluster 2024-03-07T03:11:39.404954+0000 mon.a (mon.0) 910 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:40.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:40 smithi189 bash[23614]: cluster 2024-03-07T03:11:40.249445+0000 mon.a (mon.0) 911 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2024-03-07T03:11:41.307 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroringWatcher.ModeUpdated (2063 ms) 2024-03-07T03:11:41.307 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroringWatcher.ImageStatusUpdated 2024-03-07T03:11:41.344 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroringWatcher.ImageStatusUpdated (37 ms) 2024-03-07T03:11:42.276 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 2 tests from TestMirroringWatcher (2100 ms total) 2024-03-07T03:11:42.276 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:11:42.277 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 57 tests from TestMigration 2024-03-07T03:11:42.492 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:11:42.513 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:42 smithi145 bash[21339]: cluster 2024-03-07T03:11:40.467141+0000 mgr.y (mgr.24302) 215 : cluster 0 pgmap v217: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:42.513 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:42 smithi145 bash[21339]: cluster 2024-03-07T03:11:41.252934+0000 mon.a (mon.0) 912 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2024-03-07T03:11:42.513 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:42 smithi145 bash[21339]: cluster 2024-03-07T03:11:41.264654+0000 mon.a (mon.0) 913 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:42.513 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:42 smithi145 bash[21339]: cluster 2024-03-07T03:11:41.264784+0000 mon.a (mon.0) 914 : cluster 1 Cluster is now healthy 2024-03-07T03:11:42.513 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:11:42 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:11:42] "GET /metrics HTTP/1.1" 200 33766 "" "Prometheus/2.43.0" 2024-03-07T03:11:42.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:42 smithi145 bash[30906]: cluster 2024-03-07T03:11:40.467141+0000 mgr.y (mgr.24302) 215 : cluster 0 pgmap v217: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:42.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:42 smithi145 bash[30906]: cluster 2024-03-07T03:11:41.252934+0000 mon.a (mon.0) 912 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2024-03-07T03:11:42.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:42 smithi145 bash[30906]: cluster 2024-03-07T03:11:41.264654+0000 mon.a (mon.0) 913 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:42.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:42 smithi145 bash[30906]: cluster 2024-03-07T03:11:41.264784+0000 mon.a (mon.0) 914 : cluster 1 Cluster is now healthy 2024-03-07T03:11:42.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:42 smithi189 bash[23614]: cluster 2024-03-07T03:11:40.467141+0000 mgr.y (mgr.24302) 215 : cluster 0 pgmap v217: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:42.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:42 smithi189 bash[23614]: cluster 2024-03-07T03:11:41.252934+0000 mon.a (mon.0) 912 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2024-03-07T03:11:42.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:42 smithi189 bash[23614]: cluster 2024-03-07T03:11:41.264654+0000 mon.a (mon.0) 913 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:42.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:42 smithi189 bash[23614]: cluster 2024-03-07T03:11:41.264784+0000 mon.a (mon.0) 914 : cluster 1 Cluster is now healthy 2024-03-07T03:11:43.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:43 smithi145 bash[21339]: cluster 2024-03-07T03:11:42.279602+0000 mon.a (mon.0) 915 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2024-03-07T03:11:43.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:43 smithi145 bash[30906]: cluster 2024-03-07T03:11:42.279602+0000 mon.a (mon.0) 915 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2024-03-07T03:11:43.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:43 smithi189 bash[23614]: cluster 2024-03-07T03:11:42.279602+0000 mon.a (mon.0) 915 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2024-03-07T03:11:44.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:44 smithi145 bash[21339]: cluster 2024-03-07T03:11:42.467520+0000 mgr.y (mgr.24302) 216 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:44.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:44 smithi145 bash[21339]: cluster 2024-03-07T03:11:43.299421+0000 mon.a (mon.0) 916 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2024-03-07T03:11:44.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:44 smithi145 bash[21339]: audit 2024-03-07T03:11:43.302680+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.145:0/3729804469' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:44.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:44 smithi145 bash[21339]: audit 2024-03-07T03:11:43.308036+0000 mon.a (mon.0) 917 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:44.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:44 smithi145 bash[30906]: cluster 2024-03-07T03:11:42.467520+0000 mgr.y (mgr.24302) 216 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:44.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:44 smithi145 bash[30906]: cluster 2024-03-07T03:11:43.299421+0000 mon.a (mon.0) 916 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2024-03-07T03:11:44.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:44 smithi145 bash[30906]: audit 2024-03-07T03:11:43.302680+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.145:0/3729804469' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:44.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:44 smithi145 bash[30906]: audit 2024-03-07T03:11:43.308036+0000 mon.a (mon.0) 917 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:44.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:44 smithi189 bash[23614]: cluster 2024-03-07T03:11:42.467520+0000 mgr.y (mgr.24302) 216 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:44.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:44 smithi189 bash[23614]: cluster 2024-03-07T03:11:43.299421+0000 mon.a (mon.0) 916 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2024-03-07T03:11:44.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:44 smithi189 bash[23614]: audit 2024-03-07T03:11:43.302680+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.145:0/3729804469' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:44.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:44 smithi189 bash[23614]: audit 2024-03-07T03:11:43.308036+0000 mon.a (mon.0) 917 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:11:45.306 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.Empty 2024-03-07T03:11:45.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:45 smithi145 bash[30906]: audit 2024-03-07T03:11:44.297895+0000 mon.a (mon.0) 918 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:45.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:45 smithi145 bash[30906]: cluster 2024-03-07T03:11:44.298378+0000 mon.a (mon.0) 919 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2024-03-07T03:11:45.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:45 smithi145 bash[21339]: audit 2024-03-07T03:11:44.297895+0000 mon.a (mon.0) 918 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:45.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:45 smithi145 bash[21339]: cluster 2024-03-07T03:11:44.298378+0000 mon.a (mon.0) 919 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2024-03-07T03:11:45.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:45 smithi189 bash[23614]: audit 2024-03-07T03:11:44.297895+0000 mon.a (mon.0) 918 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:11:45.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:45 smithi189 bash[23614]: cluster 2024-03-07T03:11:44.298378+0000 mon.a (mon.0) 919 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2024-03-07T03:11:46.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:46 smithi189 bash[23614]: cluster 2024-03-07T03:11:44.467926+0000 mgr.y (mgr.24302) 217 : cluster 0 pgmap v223: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:46.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:46 smithi189 bash[23614]: cluster 2024-03-07T03:11:45.316543+0000 mon.a (mon.0) 920 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2024-03-07T03:11:46.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:46 smithi145 bash[21339]: cluster 2024-03-07T03:11:44.467926+0000 mgr.y (mgr.24302) 217 : cluster 0 pgmap v223: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:46.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:46 smithi145 bash[21339]: cluster 2024-03-07T03:11:45.316543+0000 mon.a (mon.0) 920 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2024-03-07T03:11:46.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:46 smithi145 bash[30906]: cluster 2024-03-07T03:11:44.467926+0000 mgr.y (mgr.24302) 217 : cluster 0 pgmap v223: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:46.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:46 smithi145 bash[30906]: cluster 2024-03-07T03:11:45.316543+0000 mon.a (mon.0) 920 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2024-03-07T03:11:47.453 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:47.475 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:47.727 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:11:47.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:47 smithi145 bash[21339]: cluster 2024-03-07T03:11:46.396264+0000 mon.a (mon.0) 921 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2024-03-07T03:11:47.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:47 smithi145 bash[30906]: cluster 2024-03-07T03:11:46.396264+0000 mon.a (mon.0) 921 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2024-03-07T03:11:47.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:47 smithi189 bash[23614]: cluster 2024-03-07T03:11:46.396264+0000 mon.a (mon.0) 921 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2024-03-07T03:11:47.859 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:11:47.993 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.Empty (2687 ms) 2024-03-07T03:11:47.993 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.OtherName 2024-03-07T03:11:48.044 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:48.065 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:48.284 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:11:48.426 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:11:48.566 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.OtherName (572 ms) 2024-03-07T03:11:48.566 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.OtherPool 2024-03-07T03:11:48.614 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:48.730 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:48.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:48 smithi145 bash[30906]: cluster 2024-03-07T03:11:46.468347+0000 mgr.y (mgr.24302) 218 : cluster 0 pgmap v226: 137 pgs: 32 unknown, 32 creating+peering, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:48.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:48 smithi145 bash[30906]: cluster 2024-03-07T03:11:47.388470+0000 mon.a (mon.0) 922 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:48.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:48 smithi145 bash[30906]: cluster 2024-03-07T03:11:47.401019+0000 mon.a (mon.0) 923 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2024-03-07T03:11:48.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:48 smithi145 bash[30906]: audit 2024-03-07T03:11:47.457232+0000 mon.a (mon.0) 924 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:48.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:48 smithi145 bash[30906]: audit 2024-03-07T03:11:47.460901+0000 mon.c (mon.2) 134 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:48.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:48 smithi145 bash[21339]: cluster 2024-03-07T03:11:46.468347+0000 mgr.y (mgr.24302) 218 : cluster 0 pgmap v226: 137 pgs: 32 unknown, 32 creating+peering, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:48.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:48 smithi145 bash[21339]: cluster 2024-03-07T03:11:47.388470+0000 mon.a (mon.0) 922 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:48.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:48 smithi145 bash[21339]: cluster 2024-03-07T03:11:47.401019+0000 mon.a (mon.0) 923 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2024-03-07T03:11:48.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:48 smithi145 bash[21339]: audit 2024-03-07T03:11:47.457232+0000 mon.a (mon.0) 924 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:48.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:48 smithi145 bash[21339]: audit 2024-03-07T03:11:47.460901+0000 mon.c (mon.2) 134 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:48.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:48 smithi189 bash[23614]: cluster 2024-03-07T03:11:46.468347+0000 mgr.y (mgr.24302) 218 : cluster 0 pgmap v226: 137 pgs: 32 unknown, 32 creating+peering, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:48.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:48 smithi189 bash[23614]: cluster 2024-03-07T03:11:47.388470+0000 mon.a (mon.0) 922 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:48.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:48 smithi189 bash[23614]: cluster 2024-03-07T03:11:47.401019+0000 mon.a (mon.0) 923 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2024-03-07T03:11:48.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:48 smithi189 bash[23614]: audit 2024-03-07T03:11:47.457232+0000 mon.a (mon.0) 924 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:48.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:48 smithi189 bash[23614]: audit 2024-03-07T03:11:47.460901+0000 mon.c (mon.2) 134 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:50.596 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:11:50.716 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:11:50.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:50 smithi145 bash[21339]: cluster 2024-03-07T03:11:48.468890+0000 mgr.y (mgr.24302) 219 : cluster 0 pgmap v228: 137 pgs: 27 unknown, 23 creating+peering, 87 active+clean; 579 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 5.7 KiB/s wr, 107 op/s 2024-03-07T03:11:50.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:50 smithi145 bash[21339]: cluster 2024-03-07T03:11:49.419643+0000 mon.a (mon.0) 925 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2024-03-07T03:11:50.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:50 smithi145 bash[30906]: cluster 2024-03-07T03:11:48.468890+0000 mgr.y (mgr.24302) 219 : cluster 0 pgmap v228: 137 pgs: 27 unknown, 23 creating+peering, 87 active+clean; 579 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 5.7 KiB/s wr, 107 op/s 2024-03-07T03:11:50.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:50 smithi145 bash[30906]: cluster 2024-03-07T03:11:49.419643+0000 mon.a (mon.0) 925 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2024-03-07T03:11:50.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:50 smithi189 bash[23614]: cluster 2024-03-07T03:11:48.468890+0000 mgr.y (mgr.24302) 219 : cluster 0 pgmap v228: 137 pgs: 27 unknown, 23 creating+peering, 87 active+clean; 579 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 5.7 KiB/s wr, 107 op/s 2024-03-07T03:11:50.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:50 smithi189 bash[23614]: cluster 2024-03-07T03:11:49.419643+0000 mon.a (mon.0) 925 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2024-03-07T03:11:50.850 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.OtherPool (2284 ms) 2024-03-07T03:11:50.850 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.OtherNamespace 2024-03-07T03:11:50.901 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:50.926 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:51.150 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:11:51.283 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:11:51.427 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.OtherNamespace (577 ms) 2024-03-07T03:11:51.427 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.DataPool 2024-03-07T03:11:51.471 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:51.487 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:51.685 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:11:51.784 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:11:51.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:51 smithi145 bash[21339]: cluster 2024-03-07T03:11:50.423942+0000 mon.a (mon.0) 926 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2024-03-07T03:11:51.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:51 smithi145 bash[21339]: cluster 2024-03-07T03:11:50.469289+0000 mgr.y (mgr.24302) 220 : cluster 0 pgmap v231: 137 pgs: 137 active+clean; 579 KiB data, 251 MiB used, 715 GiB / 715 GiB avail; 303 KiB/s rd, 16 KiB/s wr, 377 op/s 2024-03-07T03:11:51.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:51 smithi145 bash[30906]: cluster 2024-03-07T03:11:50.423942+0000 mon.a (mon.0) 926 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2024-03-07T03:11:51.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:51 smithi145 bash[30906]: cluster 2024-03-07T03:11:50.469289+0000 mgr.y (mgr.24302) 220 : cluster 0 pgmap v231: 137 pgs: 137 active+clean; 579 KiB data, 251 MiB used, 715 GiB / 715 GiB avail; 303 KiB/s rd, 16 KiB/s wr, 377 op/s 2024-03-07T03:11:51.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:51 smithi189 bash[23614]: cluster 2024-03-07T03:11:50.423942+0000 mon.a (mon.0) 926 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2024-03-07T03:11:51.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:51 smithi189 bash[23614]: cluster 2024-03-07T03:11:50.469289+0000 mgr.y (mgr.24302) 220 : cluster 0 pgmap v231: 137 pgs: 137 active+clean; 579 KiB data, 251 MiB used, 715 GiB / 715 GiB avail; 303 KiB/s rd, 16 KiB/s wr, 377 op/s 2024-03-07T03:11:51.893 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.DataPool (467 ms) 2024-03-07T03:11:51.894 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.AbortAfterPrepare 2024-03-07T03:11:51.944 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:51.966 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:52.210 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:11:52.362 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.AbortAfterPrepare (468 ms) 2024-03-07T03:11:52.362 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.AbortAfterFailedPrepare 2024-03-07T03:11:52.414 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:52.434 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:52.602 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.AbortAfterFailedPrepare (240 ms) 2024-03-07T03:11:52.602 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.AbortAfterExecute 2024-03-07T03:11:52.652 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:52.675 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:52.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:11:52 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:11:52] "GET /metrics HTTP/1.1" 200 38495 "" "Prometheus/2.43.0" 2024-03-07T03:11:52.872 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:11:52.959 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:11:53.099 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.AbortAfterExecute (497 ms) 2024-03-07T03:11:53.099 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.OtherPoolAbortAfterExecute 2024-03-07T03:11:53.157 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:53.185 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:53.509 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:11:53.686 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:11:53.895 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.OtherPoolAbortAfterExecute (795 ms) 2024-03-07T03:11:53.895 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.OtherNamespaceAbortAfterExecute 2024-03-07T03:11:53.967 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:53.997 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:54.312 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:11:54.493 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:11:54.758 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.OtherNamespaceAbortAfterExecute (863 ms) 2024-03-07T03:11:54.758 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.MirroringSamePool 2024-03-07T03:11:54.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:54 smithi145 bash[21339]: cluster 2024-03-07T03:11:52.469707+0000 mgr.y (mgr.24302) 221 : cluster 0 pgmap v232: 137 pgs: 137 active+clean; 579 KiB data, 251 MiB used, 715 GiB / 715 GiB avail; 206 KiB/s rd, 11 KiB/s wr, 256 op/s 2024-03-07T03:11:54.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:54 smithi145 bash[21339]: cluster 2024-03-07T03:11:53.418979+0000 mon.a (mon.0) 927 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2024-03-07T03:11:54.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:54 smithi145 bash[30906]: cluster 2024-03-07T03:11:52.469707+0000 mgr.y (mgr.24302) 221 : cluster 0 pgmap v232: 137 pgs: 137 active+clean; 579 KiB data, 251 MiB used, 715 GiB / 715 GiB avail; 206 KiB/s rd, 11 KiB/s wr, 256 op/s 2024-03-07T03:11:54.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:54 smithi145 bash[30906]: cluster 2024-03-07T03:11:53.418979+0000 mon.a (mon.0) 927 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2024-03-07T03:11:54.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:54 smithi189 bash[23614]: cluster 2024-03-07T03:11:52.469707+0000 mgr.y (mgr.24302) 221 : cluster 0 pgmap v232: 137 pgs: 137 active+clean; 579 KiB data, 251 MiB used, 715 GiB / 715 GiB avail; 206 KiB/s rd, 11 KiB/s wr, 256 op/s 2024-03-07T03:11:54.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:54 smithi189 bash[23614]: cluster 2024-03-07T03:11:53.418979+0000 mon.a (mon.0) 927 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2024-03-07T03:11:54.837 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:54.865 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_Migration.cc:635: Skipped 2024-03-07T03:11:54.865 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:54.881 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMigration.MirroringSamePool (124 ms) 2024-03-07T03:11:54.882 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.MirroringAbort 2024-03-07T03:11:54.958 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:54.985 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_Migration.cc:653: Skipped 2024-03-07T03:11:54.986 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:55.003 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMigration.MirroringAbort (122 ms) 2024-03-07T03:11:55.004 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.MirroringOtherPoolDisabled 2024-03-07T03:11:55.083 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:55.112 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_Migration.cc:676: Skipped 2024-03-07T03:11:55.112 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:55.130 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMigration.MirroringOtherPoolDisabled (127 ms) 2024-03-07T03:11:55.131 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.MirroringOtherPoolEnabled 2024-03-07T03:11:55.209 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:55.238 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_Migration.cc:694: Skipped 2024-03-07T03:11:55.238 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:55.256 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMigration.MirroringOtherPoolEnabled (125 ms) 2024-03-07T03:11:55.256 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.MirroringPool 2024-03-07T03:11:55.334 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:55.361 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_Migration.cc:714: Skipped 2024-03-07T03:11:55.361 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:11:55.378 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMigration.MirroringPool (122 ms) 2024-03-07T03:11:55.378 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.Group 2024-03-07T03:11:55.458 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:55.507 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:55.857 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:11:56.030 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:11:56.244 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.Group (867 ms) 2024-03-07T03:11:56.245 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.GroupAbort 2024-03-07T03:11:56.331 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:56.382 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:56.693 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:11:56.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:56 smithi145 bash[21339]: cluster 2024-03-07T03:11:54.470745+0000 mgr.y (mgr.24302) 222 : cluster 0 pgmap v234: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 763 KiB/s rd, 43 KiB/s wr, 967 op/s 2024-03-07T03:11:56.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:56 smithi145 bash[30906]: cluster 2024-03-07T03:11:54.470745+0000 mgr.y (mgr.24302) 222 : cluster 0 pgmap v234: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 763 KiB/s rd, 43 KiB/s wr, 967 op/s 2024-03-07T03:11:56.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:56 smithi189 bash[23614]: cluster 2024-03-07T03:11:54.470745+0000 mgr.y (mgr.24302) 222 : cluster 0 pgmap v234: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 763 KiB/s rd, 43 KiB/s wr, 967 op/s 2024-03-07T03:11:56.912 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.GroupAbort (667 ms) 2024-03-07T03:11:56.912 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.NoSnaps 2024-03-07T03:11:56.971 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:56.995 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:11:57.097 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:11:57.172 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:57.582 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:11:57.939 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:11:58.173 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.NoSnaps (1261 ms) 2024-03-07T03:11:58.173 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.NoSnapsOtherPool 2024-03-07T03:11:58.252 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:58.277 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:11:58.365 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:11:58.453 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:11:58.464 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:11:58.530 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:58.586 INFO:teuthology.orchestra.run.smithi145.stdout:wait for minority of mons to upgrade 2024-03-07T03:11:58.799 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:11:58.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:58 smithi145 bash[21339]: cluster 2024-03-07T03:11:56.471414+0000 mgr.y (mgr.24302) 223 : cluster 0 pgmap v235: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 526 KiB/s rd, 31 KiB/s wr, 669 op/s 2024-03-07T03:11:58.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:58 smithi145 bash[21339]: audit 2024-03-07T03:11:57.865533+0000 mon.a (mon.0) 928 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:58.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:58 smithi145 bash[30906]: cluster 2024-03-07T03:11:56.471414+0000 mgr.y (mgr.24302) 223 : cluster 0 pgmap v235: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 526 KiB/s rd, 31 KiB/s wr, 669 op/s 2024-03-07T03:11:58.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:58 smithi145 bash[30906]: audit 2024-03-07T03:11:57.865533+0000 mon.a (mon.0) 928 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:58.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:58 smithi189 bash[23614]: cluster 2024-03-07T03:11:56.471414+0000 mgr.y (mgr.24302) 223 : cluster 0 pgmap v235: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 526 KiB/s rd, 31 KiB/s wr, 669 op/s 2024-03-07T03:11:58.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:58 smithi189 bash[23614]: audit 2024-03-07T03:11:57.865533+0000 mon.a (mon.0) 928 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:11:58.978 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:11:59.212 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.NoSnapsOtherPool (1040 ms) 2024-03-07T03:11:59.213 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.NoSnapsDataPool 2024-03-07T03:11:59.295 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:11:59.319 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:11:59.429 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:11:59.503 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:11:59.742 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:59 smithi145 bash[21339]: cluster 2024-03-07T03:11:58.419649+0000 mon.a (mon.0) 929 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2024-03-07T03:11:59.742 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:11:59 smithi145 bash[21339]: cluster 2024-03-07T03:11:58.472206+0000 mgr.y (mgr.24302) 224 : cluster 0 pgmap v237: 137 pgs: 137 active+clean; 8.6 MiB data, 271 MiB used, 715 GiB / 715 GiB avail; 632 KiB/s rd, 1.0 MiB/s wr, 809 op/s 2024-03-07T03:11:59.742 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:59 smithi145 bash[30906]: cluster 2024-03-07T03:11:58.419649+0000 mon.a (mon.0) 929 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2024-03-07T03:11:59.742 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:11:59 smithi145 bash[30906]: cluster 2024-03-07T03:11:58.472206+0000 mgr.y (mgr.24302) 224 : cluster 0 pgmap v237: 137 pgs: 137 active+clean; 8.6 MiB data, 271 MiB used, 715 GiB / 715 GiB avail; 632 KiB/s rd, 1.0 MiB/s wr, 809 op/s 2024-03-07T03:11:59.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:59 smithi189 bash[23614]: cluster 2024-03-07T03:11:58.419649+0000 mon.a (mon.0) 929 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2024-03-07T03:11:59.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:11:59 smithi189 bash[23614]: cluster 2024-03-07T03:11:58.472206+0000 mgr.y (mgr.24302) 224 : cluster 0 pgmap v237: 137 pgs: 137 active+clean; 8.6 MiB data, 271 MiB used, 715 GiB / 715 GiB avail; 632 KiB/s rd, 1.0 MiB/s wr, 809 op/s 2024-03-07T03:11:59.943 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:12:00.180 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:12:00.374 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-03-07T03:12:00.434 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.NoSnapsDataPool (1221 ms) 2024-03-07T03:12:00.434 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.NoSnapsShrinkAfterPrepare 2024-03-07T03:12:00.491 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:00.510 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:12:00.569 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:12:00.632 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:12:00.945 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 41943040 2024-03-07T03:12:00.965 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:12:01.172 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:12:01.392 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.NoSnapsShrinkAfterPrepare (958 ms) 2024-03-07T03:12:01.392 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroBeforePrepare 2024-03-07T03:12:01.475 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:01.499 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:12:01.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:01 smithi189 bash[23614]: audit 2024-03-07T03:12:00.470262+0000 mon.a (mon.0) 930 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:01.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:01 smithi189 bash[23614]: cephadm 2024-03-07T03:12:00.471599+0000 mgr.y (mgr.24302) 225 : cephadm 1 Upgrade: Target is version 19.0.0-1905-gea9a5688 (unknown) 2024-03-07T03:12:01.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:01 smithi189 bash[23614]: cephadm 2024-03-07T03:12:00.471711+0000 mgr.y (mgr.24302) 226 : cephadm 1 Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71, digests ['quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71'] 2024-03-07T03:12:01.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:01 smithi189 bash[23614]: cluster 2024-03-07T03:12:00.473015+0000 mgr.y (mgr.24302) 227 : cluster 0 pgmap v238: 137 pgs: 137 active+clean; 65 MiB data, 356 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 10 MiB/s wr, 1.19k op/s 2024-03-07T03:12:01.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:01 smithi189 bash[23614]: audit 2024-03-07T03:12:00.474401+0000 mon.c (mon.2) 135 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:12:01.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:01 smithi189 bash[23614]: audit 2024-03-07T03:12:00.481839+0000 mon.a (mon.0) 931 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:01.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:01 smithi189 bash[23614]: cephadm 2024-03-07T03:12:00.483834+0000 mgr.y (mgr.24302) 228 : cephadm 1 Upgrade: Need to upgrade myself (mgr.y) 2024-03-07T03:12:01.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:01 smithi189 bash[23614]: cephadm 2024-03-07T03:12:00.849276+0000 mgr.y (mgr.24302) 229 : cephadm 1 Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 on smithi189 2024-03-07T03:12:01.588 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:12:01.657 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 0 2024-03-07T03:12:01.692 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:12:01.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[21339]: audit 2024-03-07T03:12:00.470262+0000 mon.a (mon.0) 930 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:01.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[21339]: cephadm 2024-03-07T03:12:00.471599+0000 mgr.y (mgr.24302) 225 : cephadm 1 Upgrade: Target is version 19.0.0-1905-gea9a5688 (unknown) 2024-03-07T03:12:01.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[21339]: cephadm 2024-03-07T03:12:00.471711+0000 mgr.y (mgr.24302) 226 : cephadm 1 Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71, digests ['quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71'] 2024-03-07T03:12:01.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[21339]: cluster 2024-03-07T03:12:00.473015+0000 mgr.y (mgr.24302) 227 : cluster 0 pgmap v238: 137 pgs: 137 active+clean; 65 MiB data, 356 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 10 MiB/s wr, 1.19k op/s 2024-03-07T03:12:01.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[21339]: audit 2024-03-07T03:12:00.474401+0000 mon.c (mon.2) 135 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:12:01.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[21339]: audit 2024-03-07T03:12:00.481839+0000 mon.a (mon.0) 931 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:01.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[21339]: cephadm 2024-03-07T03:12:00.483834+0000 mgr.y (mgr.24302) 228 : cephadm 1 Upgrade: Need to upgrade myself (mgr.y) 2024-03-07T03:12:01.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[21339]: cephadm 2024-03-07T03:12:00.849276+0000 mgr.y (mgr.24302) 229 : cephadm 1 Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 on smithi189 2024-03-07T03:12:01.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[30906]: audit 2024-03-07T03:12:00.470262+0000 mon.a (mon.0) 930 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:01.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[30906]: cephadm 2024-03-07T03:12:00.471599+0000 mgr.y (mgr.24302) 225 : cephadm 1 Upgrade: Target is version 19.0.0-1905-gea9a5688 (unknown) 2024-03-07T03:12:01.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[30906]: cephadm 2024-03-07T03:12:00.471711+0000 mgr.y (mgr.24302) 226 : cephadm 1 Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71, digests ['quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71'] 2024-03-07T03:12:01.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[30906]: cluster 2024-03-07T03:12:00.473015+0000 mgr.y (mgr.24302) 227 : cluster 0 pgmap v238: 137 pgs: 137 active+clean; 65 MiB data, 356 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 10 MiB/s wr, 1.19k op/s 2024-03-07T03:12:01.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[30906]: audit 2024-03-07T03:12:00.474401+0000 mon.c (mon.2) 135 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:12:01.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[30906]: audit 2024-03-07T03:12:00.481839+0000 mon.a (mon.0) 931 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:01.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[30906]: cephadm 2024-03-07T03:12:00.483834+0000 mgr.y (mgr.24302) 228 : cephadm 1 Upgrade: Need to upgrade myself (mgr.y) 2024-03-07T03:12:01.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:01 smithi145 bash[30906]: cephadm 2024-03-07T03:12:00.849276+0000 mgr.y (mgr.24302) 229 : cephadm 1 Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 on smithi189 2024-03-07T03:12:01.963 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:12:02.148 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:12:02.276 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroBeforePrepare (885 ms) 2024-03-07T03:12:02.277 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroAfterPrepare 2024-03-07T03:12:02.356 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:02.385 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:12:02.479 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:12:02.543 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:12:02.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:02 smithi145 bash[21339]: audit 2024-03-07T03:12:02.454129+0000 mon.c (mon.2) 136 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:02.811 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:12:02 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:12:02] "GET /metrics HTTP/1.1" 200 38495 "" "Prometheus/2.43.0" 2024-03-07T03:12:02.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:02 smithi145 bash[30906]: audit 2024-03-07T03:12:02.454129+0000 mon.c (mon.2) 136 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:02.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:02 smithi189 bash[23614]: audit 2024-03-07T03:12:02.454129+0000 mon.c (mon.2) 136 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:02.923 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 0 2024-03-07T03:12:02.957 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:12:03.122 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:12:03.216 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroAfterPrepare (940 ms) 2024-03-07T03:12:03.216 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.NoSnapsExpandAfterPrepare 2024-03-07T03:12:03.269 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:03.288 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:12:03.362 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:12:03.443 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:12:03.871 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 167772160 2024-03-07T03:12:03.893 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:12:04.127 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:12:04.396 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.NoSnapsExpandAfterPrepare (1179 ms) 2024-03-07T03:12:04.396 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.NoSnapsSnapAfterPrepare 2024-03-07T03:12:04.478 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:04.507 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:12:04.603 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:12:04.670 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:12:04.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:04 smithi145 bash[21339]: cluster 2024-03-07T03:12:02.473875+0000 mgr.y (mgr.24302) 230 : cluster 0 pgmap v239: 137 pgs: 137 active+clean; 65 MiB data, 356 MiB used, 715 GiB / 715 GiB avail; 9.6 MiB/s rd, 8.9 MiB/s wr, 1.05k op/s 2024-03-07T03:12:04.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:04 smithi145 bash[30906]: cluster 2024-03-07T03:12:02.473875+0000 mgr.y (mgr.24302) 230 : cluster 0 pgmap v239: 137 pgs: 137 active+clean; 65 MiB data, 356 MiB used, 715 GiB / 715 GiB avail; 9.6 MiB/s rd, 8.9 MiB/s wr, 1.05k op/s 2024-03-07T03:12:04.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:04 smithi189 bash[23614]: cluster 2024-03-07T03:12:02.473875+0000 mgr.y (mgr.24302) 230 : cluster 0 pgmap v239: 137 pgs: 137 active+clean; 65 MiB data, 356 MiB used, 715 GiB / 715 GiB avail; 9.6 MiB/s rd, 8.9 MiB/s wr, 1.05k op/s 2024-03-07T03:12:05.087 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: after_prepare_snap 2024-03-07T03:12:05.141 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:06.448 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 41943040 2024-03-07T03:12:06.498 INFO:tasks.workunit.client.0.smithi145.stdout:write: * 0~1000 2024-03-07T03:12:06.511 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:12:06.668 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:06 smithi145 bash[21339]: cluster 2024-03-07T03:12:04.475545+0000 mgr.y (mgr.24302) 231 : cluster 0 pgmap v240: 137 pgs: 137 active+clean; 161 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 36 MiB/s rd, 32 MiB/s wr, 1.03k op/s 2024-03-07T03:12:06.668 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:06 smithi145 bash[21339]: cluster 2024-03-07T03:12:05.434186+0000 mon.a (mon.0) 932 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2024-03-07T03:12:06.668 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:06 smithi145 bash[30906]: cluster 2024-03-07T03:12:04.475545+0000 mgr.y (mgr.24302) 231 : cluster 0 pgmap v240: 137 pgs: 137 active+clean; 161 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 36 MiB/s rd, 32 MiB/s wr, 1.03k op/s 2024-03-07T03:12:06.669 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:06 smithi145 bash[30906]: cluster 2024-03-07T03:12:05.434186+0000 mon.a (mon.0) 932 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2024-03-07T03:12:06.695 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:12:06.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:06 smithi189 bash[23614]: cluster 2024-03-07T03:12:04.475545+0000 mgr.y (mgr.24302) 231 : cluster 0 pgmap v240: 137 pgs: 137 active+clean; 161 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 36 MiB/s rd, 32 MiB/s wr, 1.03k op/s 2024-03-07T03:12:06.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:06 smithi189 bash[23614]: cluster 2024-03-07T03:12:05.434186+0000 mon.a (mon.0) 932 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2024-03-07T03:12:06.906 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.NoSnapsSnapAfterPrepare (2510 ms) 2024-03-07T03:12:06.906 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.Snaps 2024-03-07T03:12:06.962 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:06.987 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:12:07.067 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:12:07.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:07 smithi145 bash[21339]: cluster 2024-03-07T03:12:06.438254+0000 mon.a (mon.0) 933 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2024-03-07T03:12:07.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:07 smithi145 bash[21339]: cluster 2024-03-07T03:12:06.476392+0000 mgr.y (mgr.24302) 232 : cluster 0 pgmap v243: 137 pgs: 137 active+clean; 161 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 45 MiB/s rd, 39 MiB/s wr, 1.07k op/s 2024-03-07T03:12:07.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:07 smithi145 bash[30906]: cluster 2024-03-07T03:12:06.438254+0000 mon.a (mon.0) 933 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2024-03-07T03:12:07.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:07 smithi145 bash[30906]: cluster 2024-03-07T03:12:06.476392+0000 mgr.y (mgr.24302) 232 : cluster 0 pgmap v243: 137 pgs: 137 active+clean; 161 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 45 MiB/s rd, 39 MiB/s wr, 1.07k op/s 2024-03-07T03:12:07.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:07 smithi189 bash[23614]: cluster 2024-03-07T03:12:06.438254+0000 mon.a (mon.0) 933 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2024-03-07T03:12:07.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:07 smithi189 bash[23614]: cluster 2024-03-07T03:12:06.476392+0000 mgr.y (mgr.24302) 232 : cluster 0 pgmap v243: 137 pgs: 137 active+clean; 161 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 45 MiB/s rd, 39 MiB/s wr, 1.07k op/s 2024-03-07T03:12:08.472 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 8388609~8388609 2024-03-07T03:12:08.502 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 0~1000 2024-03-07T03:12:08.502 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 1010~1000 2024-03-07T03:12:08.502 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:12:08.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:08 smithi145 bash[30906]: cluster 2024-03-07T03:12:07.448560+0000 mon.a (mon.0) 934 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2024-03-07T03:12:08.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:08 smithi145 bash[21339]: cluster 2024-03-07T03:12:07.448560+0000 mon.a (mon.0) 934 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2024-03-07T03:12:08.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:08 smithi189 bash[23614]: cluster 2024-03-07T03:12:07.448560+0000 mon.a (mon.0) 934 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2024-03-07T03:12:09.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:09 smithi145 bash[21339]: cluster 2024-03-07T03:12:08.450203+0000 mon.a (mon.0) 935 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2024-03-07T03:12:09.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:09 smithi145 bash[21339]: cluster 2024-03-07T03:12:08.477698+0000 mgr.y (mgr.24302) 233 : cluster 0 pgmap v246: 137 pgs: 137 active+clean; 197 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 12 MiB/s wr, 371 op/s 2024-03-07T03:12:09.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:09 smithi145 bash[21339]: audit 2024-03-07T03:12:09.080100+0000 mon.c (mon.2) 137 : audit 0 from='client.? 172.21.15.145:0/1639076357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:09.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:09 smithi145 bash[30906]: cluster 2024-03-07T03:12:08.450203+0000 mon.a (mon.0) 935 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2024-03-07T03:12:09.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:09 smithi145 bash[30906]: cluster 2024-03-07T03:12:08.477698+0000 mgr.y (mgr.24302) 233 : cluster 0 pgmap v246: 137 pgs: 137 active+clean; 197 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 12 MiB/s wr, 371 op/s 2024-03-07T03:12:09.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:09 smithi145 bash[30906]: audit 2024-03-07T03:12:09.080100+0000 mon.c (mon.2) 137 : audit 0 from='client.? 172.21.15.145:0/1639076357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:09.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:09 smithi189 bash[23614]: cluster 2024-03-07T03:12:08.450203+0000 mon.a (mon.0) 935 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2024-03-07T03:12:09.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:09 smithi189 bash[23614]: cluster 2024-03-07T03:12:08.477698+0000 mgr.y (mgr.24302) 233 : cluster 0 pgmap v246: 137 pgs: 137 active+clean; 197 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 12 MiB/s wr, 371 op/s 2024-03-07T03:12:09.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:09 smithi189 bash[23614]: audit 2024-03-07T03:12:09.080100+0000 mon.c (mon.2) 137 : audit 0 from='client.? 172.21.15.145:0/1639076357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:10.502 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 8388609~1000 2024-03-07T03:12:10.522 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 16777228~1000 2024-03-07T03:12:10.522 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 167772160 2024-03-07T03:12:10.541 INFO:tasks.workunit.client.0.smithi145.stdout:write: 2 83886079~8388609 2024-03-07T03:12:10.585 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:12:10.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:10 smithi145 bash[21339]: cluster 2024-03-07T03:12:09.475269+0000 mon.a (mon.0) 936 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2024-03-07T03:12:10.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:10 smithi145 bash[30906]: cluster 2024-03-07T03:12:09.475269+0000 mon.a (mon.0) 936 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2024-03-07T03:12:10.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:10 smithi189 bash[23614]: cluster 2024-03-07T03:12:09.475269+0000 mon.a (mon.0) 936 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2024-03-07T03:12:11.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:11 smithi145 bash[21339]: cluster 2024-03-07T03:12:10.476832+0000 mon.a (mon.0) 937 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2024-03-07T03:12:11.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:11 smithi145 bash[21339]: cluster 2024-03-07T03:12:10.478704+0000 mgr.y (mgr.24302) 234 : cluster 0 pgmap v249: 137 pgs: 137 active+clean; 209 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 48 MiB/s rd, 20 MiB/s wr, 638 op/s 2024-03-07T03:12:11.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:11 smithi145 bash[30906]: cluster 2024-03-07T03:12:10.476832+0000 mon.a (mon.0) 937 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2024-03-07T03:12:11.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:11 smithi145 bash[30906]: cluster 2024-03-07T03:12:10.478704+0000 mgr.y (mgr.24302) 234 : cluster 0 pgmap v249: 137 pgs: 137 active+clean; 209 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 48 MiB/s rd, 20 MiB/s wr, 638 op/s 2024-03-07T03:12:11.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:11 smithi189 bash[23614]: cluster 2024-03-07T03:12:10.476832+0000 mon.a (mon.0) 937 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2024-03-07T03:12:11.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:11 smithi189 bash[23614]: cluster 2024-03-07T03:12:10.478704+0000 mgr.y (mgr.24302) 234 : cluster 0 pgmap v249: 137 pgs: 137 active+clean; 209 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 48 MiB/s rd, 20 MiB/s wr, 638 op/s 2024-03-07T03:12:12.517 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:12.569 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 83886079~1 2024-03-07T03:12:12.588 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:12:12.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:12:12 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:12:12] "GET /metrics HTTP/1.1" 200 38597 "" "Prometheus/2.43.0" 2024-03-07T03:12:12.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:12 smithi145 bash[30906]: cluster 2024-03-07T03:12:11.497197+0000 mon.a (mon.0) 938 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2024-03-07T03:12:12.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:12 smithi145 bash[30906]: audit 2024-03-07T03:12:11.525753+0000 mon.a (mon.0) 939 : audit 0 from='client.? 172.21.15.145:0/938898602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:12.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:12 smithi145 bash[21339]: cluster 2024-03-07T03:12:11.497197+0000 mon.a (mon.0) 938 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2024-03-07T03:12:12.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:12 smithi145 bash[21339]: audit 2024-03-07T03:12:11.525753+0000 mon.a (mon.0) 939 : audit 0 from='client.? 172.21.15.145:0/938898602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:12.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:12 smithi189 bash[23614]: cluster 2024-03-07T03:12:11.497197+0000 mon.a (mon.0) 938 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2024-03-07T03:12:12.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:12 smithi189 bash[23614]: audit 2024-03-07T03:12:11.525753+0000 mon.a (mon.0) 939 : audit 0 from='client.? 172.21.15.145:0/938898602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:13.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:13 smithi145 bash[21339]: cluster 2024-03-07T03:12:12.479467+0000 mgr.y (mgr.24302) 235 : cluster 0 pgmap v251: 137 pgs: 137 active+clean; 209 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 48 MiB/s rd, 20 MiB/s wr, 633 op/s 2024-03-07T03:12:13.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:13 smithi145 bash[21339]: cluster 2024-03-07T03:12:12.500323+0000 mon.a (mon.0) 940 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2024-03-07T03:12:13.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:13 smithi145 bash[30906]: cluster 2024-03-07T03:12:12.479467+0000 mgr.y (mgr.24302) 235 : cluster 0 pgmap v251: 137 pgs: 137 active+clean; 209 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 48 MiB/s rd, 20 MiB/s wr, 633 op/s 2024-03-07T03:12:13.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:13 smithi145 bash[30906]: cluster 2024-03-07T03:12:12.500323+0000 mon.a (mon.0) 940 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2024-03-07T03:12:13.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:13 smithi189 bash[23614]: cluster 2024-03-07T03:12:12.479467+0000 mgr.y (mgr.24302) 235 : cluster 0 pgmap v251: 137 pgs: 137 active+clean; 209 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 48 MiB/s rd, 20 MiB/s wr, 633 op/s 2024-03-07T03:12:13.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:13 smithi189 bash[23614]: cluster 2024-03-07T03:12:12.500323+0000 mon.a (mon.0) 940 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2024-03-07T03:12:14.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:14 smithi145 bash[21339]: cluster 2024-03-07T03:12:13.525948+0000 mon.a (mon.0) 941 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2024-03-07T03:12:14.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:14 smithi145 bash[21339]: audit 2024-03-07T03:12:13.958985+0000 mon.a (mon.0) 942 : audit 0 from='client.? 172.21.15.145:0/3687346756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:14.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:14 smithi145 bash[30906]: cluster 2024-03-07T03:12:13.525948+0000 mon.a (mon.0) 941 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2024-03-07T03:12:14.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:14 smithi145 bash[30906]: audit 2024-03-07T03:12:13.958985+0000 mon.a (mon.0) 942 : audit 0 from='client.? 172.21.15.145:0/3687346756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:14 smithi189 bash[23614]: cluster 2024-03-07T03:12:13.525948+0000 mon.a (mon.0) 941 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2024-03-07T03:12:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:14 smithi189 bash[23614]: audit 2024-03-07T03:12:13.958985+0000 mon.a (mon.0) 942 : audit 0 from='client.? 172.21.15.145:0/3687346756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:15.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:15 smithi145 bash[30906]: cluster 2024-03-07T03:12:14.480389+0000 mgr.y (mgr.24302) 236 : cluster 0 pgmap v254: 137 pgs: 137 active+clean; 249 MiB data, 997 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 8.0 MiB/s wr, 198 op/s 2024-03-07T03:12:15.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:15 smithi145 bash[30906]: cluster 2024-03-07T03:12:14.539785+0000 mon.a (mon.0) 943 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2024-03-07T03:12:15.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:15 smithi145 bash[21339]: cluster 2024-03-07T03:12:14.480389+0000 mgr.y (mgr.24302) 236 : cluster 0 pgmap v254: 137 pgs: 137 active+clean; 249 MiB data, 997 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 8.0 MiB/s wr, 198 op/s 2024-03-07T03:12:15.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:15 smithi145 bash[21339]: cluster 2024-03-07T03:12:14.539785+0000 mon.a (mon.0) 943 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2024-03-07T03:12:15.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:15 smithi189 bash[23614]: cluster 2024-03-07T03:12:14.480389+0000 mgr.y (mgr.24302) 236 : cluster 0 pgmap v254: 137 pgs: 137 active+clean; 249 MiB data, 997 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 8.0 MiB/s wr, 198 op/s 2024-03-07T03:12:15.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:15 smithi189 bash[23614]: cluster 2024-03-07T03:12:14.539785+0000 mon.a (mon.0) 943 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2024-03-07T03:12:16.384 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:12:16.601 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:12:16.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:16 smithi145 bash[21339]: cluster 2024-03-07T03:12:15.546494+0000 mon.a (mon.0) 944 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2024-03-07T03:12:16.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:16 smithi145 bash[21339]: audit 2024-03-07T03:12:16.434805+0000 mon.a (mon.0) 945 : audit 0 from='client.? 172.21.15.145:0/4010397324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:16.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:16 smithi145 bash[30906]: cluster 2024-03-07T03:12:15.546494+0000 mon.a (mon.0) 944 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2024-03-07T03:12:16.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:16 smithi145 bash[30906]: audit 2024-03-07T03:12:16.434805+0000 mon.a (mon.0) 945 : audit 0 from='client.? 172.21.15.145:0/4010397324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:16.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:16 smithi189 bash[23614]: cluster 2024-03-07T03:12:15.546494+0000 mon.a (mon.0) 944 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2024-03-07T03:12:16.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:16 smithi189 bash[23614]: audit 2024-03-07T03:12:16.434805+0000 mon.a (mon.0) 945 : audit 0 from='client.? 172.21.15.145:0/4010397324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:17.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:17 smithi189 bash[23614]: cluster 2024-03-07T03:12:16.481243+0000 mgr.y (mgr.24302) 237 : cluster 0 pgmap v257: 137 pgs: 137 active+clean; 249 MiB data, 997 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 8.0 MiB/s wr, 198 op/s 2024-03-07T03:12:17.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:17 smithi189 bash[23614]: audit 2024-03-07T03:12:17.454829+0000 mon.c (mon.2) 138 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:17 smithi145 bash[21339]: cluster 2024-03-07T03:12:16.481243+0000 mgr.y (mgr.24302) 237 : cluster 0 pgmap v257: 137 pgs: 137 active+clean; 249 MiB data, 997 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 8.0 MiB/s wr, 198 op/s 2024-03-07T03:12:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:17 smithi145 bash[21339]: audit 2024-03-07T03:12:17.454829+0000 mon.c (mon.2) 138 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:18.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:17 smithi145 bash[30906]: cluster 2024-03-07T03:12:16.481243+0000 mgr.y (mgr.24302) 237 : cluster 0 pgmap v257: 137 pgs: 137 active+clean; 249 MiB data, 997 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 8.0 MiB/s wr, 198 op/s 2024-03-07T03:12:18.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:17 smithi145 bash[30906]: audit 2024-03-07T03:12:17.454829+0000 mon.c (mon.2) 138 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:18.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:18 smithi189 bash[23614]: cluster 2024-03-07T03:12:17.572679+0000 mon.a (mon.0) 946 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2024-03-07T03:12:19.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:18 smithi145 bash[21339]: cluster 2024-03-07T03:12:17.572679+0000 mon.a (mon.0) 946 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2024-03-07T03:12:19.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:18 smithi145 bash[30906]: cluster 2024-03-07T03:12:17.572679+0000 mon.a (mon.0) 946 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2024-03-07T03:12:19.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:19 smithi189 bash[23614]: cluster 2024-03-07T03:12:18.482162+0000 mgr.y (mgr.24302) 238 : cluster 0 pgmap v259: 137 pgs: 137 active+clean; 265 MiB data, 1022 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.2 MiB/s wr, 282 op/s 2024-03-07T03:12:19.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:19 smithi189 bash[23614]: cluster 2024-03-07T03:12:18.591682+0000 mon.a (mon.0) 947 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2024-03-07T03:12:19.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:19 smithi189 bash[23614]: audit 2024-03-07T03:12:18.886441+0000 mon.a (mon.0) 948 : audit 0 from='client.? 172.21.15.145:0/1579814241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:20.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:19 smithi145 bash[30906]: cluster 2024-03-07T03:12:18.482162+0000 mgr.y (mgr.24302) 238 : cluster 0 pgmap v259: 137 pgs: 137 active+clean; 265 MiB data, 1022 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.2 MiB/s wr, 282 op/s 2024-03-07T03:12:20.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:19 smithi145 bash[30906]: cluster 2024-03-07T03:12:18.591682+0000 mon.a (mon.0) 947 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2024-03-07T03:12:20.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:19 smithi145 bash[30906]: audit 2024-03-07T03:12:18.886441+0000 mon.a (mon.0) 948 : audit 0 from='client.? 172.21.15.145:0/1579814241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:20.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:19 smithi145 bash[21339]: cluster 2024-03-07T03:12:18.482162+0000 mgr.y (mgr.24302) 238 : cluster 0 pgmap v259: 137 pgs: 137 active+clean; 265 MiB data, 1022 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.2 MiB/s wr, 282 op/s 2024-03-07T03:12:20.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:19 smithi145 bash[21339]: cluster 2024-03-07T03:12:18.591682+0000 mon.a (mon.0) 947 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2024-03-07T03:12:20.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:19 smithi145 bash[21339]: audit 2024-03-07T03:12:18.886441+0000 mon.a (mon.0) 948 : audit 0 from='client.? 172.21.15.145:0/1579814241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:20.148 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.Snaps (13243 ms) 2024-03-07T03:12:20.148 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.SnapsOtherPool 2024-03-07T03:12:20.218 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:20.243 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:12:20.337 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:12:21.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:20 smithi145 bash[21339]: cluster 2024-03-07T03:12:19.604750+0000 mon.a (mon.0) 949 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2024-03-07T03:12:21.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:20 smithi145 bash[30906]: cluster 2024-03-07T03:12:19.604750+0000 mon.a (mon.0) 949 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2024-03-07T03:12:21.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:20 smithi189 bash[23614]: cluster 2024-03-07T03:12:19.604750+0000 mon.a (mon.0) 949 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2024-03-07T03:12:21.632 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 8388609~8388609 2024-03-07T03:12:21.667 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 0~1000 2024-03-07T03:12:21.667 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 1010~1000 2024-03-07T03:12:21.667 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:12:22.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:21 smithi145 bash[21339]: cluster 2024-03-07T03:12:20.482822+0000 mgr.y (mgr.24302) 239 : cluster 0 pgmap v262: 137 pgs: 137 active+clean; 269 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 4.9 MiB/s wr, 321 op/s 2024-03-07T03:12:22.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:21 smithi145 bash[21339]: cluster 2024-03-07T03:12:20.619694+0000 mon.a (mon.0) 950 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2024-03-07T03:12:22.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:21 smithi145 bash[21339]: audit 2024-03-07T03:12:21.322794+0000 mon.a (mon.0) 951 : audit 0 from='client.? 172.21.15.145:0/3497135028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:22.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:21 smithi145 bash[30906]: cluster 2024-03-07T03:12:20.482822+0000 mgr.y (mgr.24302) 239 : cluster 0 pgmap v262: 137 pgs: 137 active+clean; 269 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 4.9 MiB/s wr, 321 op/s 2024-03-07T03:12:22.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:21 smithi145 bash[30906]: cluster 2024-03-07T03:12:20.619694+0000 mon.a (mon.0) 950 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2024-03-07T03:12:22.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:21 smithi145 bash[30906]: audit 2024-03-07T03:12:21.322794+0000 mon.a (mon.0) 951 : audit 0 from='client.? 172.21.15.145:0/3497135028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:22.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:21 smithi189 bash[23614]: cluster 2024-03-07T03:12:20.482822+0000 mgr.y (mgr.24302) 239 : cluster 0 pgmap v262: 137 pgs: 137 active+clean; 269 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 4.9 MiB/s wr, 321 op/s 2024-03-07T03:12:22.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:21 smithi189 bash[23614]: cluster 2024-03-07T03:12:20.619694+0000 mon.a (mon.0) 950 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2024-03-07T03:12:22.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:21 smithi189 bash[23614]: audit 2024-03-07T03:12:21.322794+0000 mon.a (mon.0) 951 : audit 0 from='client.? 172.21.15.145:0/3497135028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:22.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:22 smithi145 bash[21339]: cluster 2024-03-07T03:12:21.626376+0000 mon.a (mon.0) 952 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2024-03-07T03:12:22.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:12:22 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:12:22] "GET /metrics HTTP/1.1" 200 38625 "" "Prometheus/2.43.0" 2024-03-07T03:12:22.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:22 smithi145 bash[30906]: cluster 2024-03-07T03:12:21.626376+0000 mon.a (mon.0) 952 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2024-03-07T03:12:23.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:22 smithi189 bash[23614]: cluster 2024-03-07T03:12:21.626376+0000 mon.a (mon.0) 952 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2024-03-07T03:12:23.442 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 8388609~1000 2024-03-07T03:12:23.451 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 16777228~1000 2024-03-07T03:12:23.451 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 167772160 2024-03-07T03:12:23.463 INFO:tasks.workunit.client.0.smithi145.stdout:write: 2 83886079~8388609 2024-03-07T03:12:23.485 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:12:24.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:23 smithi145 bash[30906]: cluster 2024-03-07T03:12:22.483634+0000 mgr.y (mgr.24302) 240 : cluster 0 pgmap v265: 137 pgs: 137 active+clean; 269 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 47 op/s 2024-03-07T03:12:24.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:23 smithi145 bash[30906]: cluster 2024-03-07T03:12:22.635676+0000 mon.a (mon.0) 953 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2024-03-07T03:12:24.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:23 smithi145 bash[30906]: cluster 2024-03-07T03:12:23.428240+0000 mon.a (mon.0) 954 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2024-03-07T03:12:24.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:23 smithi145 bash[21339]: cluster 2024-03-07T03:12:22.483634+0000 mgr.y (mgr.24302) 240 : cluster 0 pgmap v265: 137 pgs: 137 active+clean; 269 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 47 op/s 2024-03-07T03:12:24.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:23 smithi145 bash[21339]: cluster 2024-03-07T03:12:22.635676+0000 mon.a (mon.0) 953 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2024-03-07T03:12:24.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:23 smithi145 bash[21339]: cluster 2024-03-07T03:12:23.428240+0000 mon.a (mon.0) 954 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2024-03-07T03:12:24.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:23 smithi189 bash[23614]: cluster 2024-03-07T03:12:22.483634+0000 mgr.y (mgr.24302) 240 : cluster 0 pgmap v265: 137 pgs: 137 active+clean; 269 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 47 op/s 2024-03-07T03:12:24.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:23 smithi189 bash[23614]: cluster 2024-03-07T03:12:22.635676+0000 mon.a (mon.0) 953 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2024-03-07T03:12:24.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:23 smithi189 bash[23614]: cluster 2024-03-07T03:12:23.428240+0000 mon.a (mon.0) 954 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2024-03-07T03:12:25.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:24 smithi145 bash[21339]: audit 2024-03-07T03:12:23.784676+0000 mon.a (mon.0) 955 : audit 0 from='client.? 172.21.15.145:0/2292692355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:25.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:24 smithi145 bash[21339]: cluster 2024-03-07T03:12:24.440244+0000 mon.a (mon.0) 956 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2024-03-07T03:12:25.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:24 smithi145 bash[30906]: audit 2024-03-07T03:12:23.784676+0000 mon.a (mon.0) 955 : audit 0 from='client.? 172.21.15.145:0/2292692355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:25.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:24 smithi145 bash[30906]: cluster 2024-03-07T03:12:24.440244+0000 mon.a (mon.0) 956 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2024-03-07T03:12:25.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:24 smithi189 bash[23614]: audit 2024-03-07T03:12:23.784676+0000 mon.a (mon.0) 955 : audit 0 from='client.? 172.21.15.145:0/2292692355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:25.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:24 smithi189 bash[23614]: cluster 2024-03-07T03:12:24.440244+0000 mon.a (mon.0) 956 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2024-03-07T03:12:25.457 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:25.500 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 83886079~1 2024-03-07T03:12:25.514 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:12:25.546 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:12:25.626 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:12:26.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:25 smithi145 bash[21339]: cluster 2024-03-07T03:12:24.484380+0000 mgr.y (mgr.24302) 241 : cluster 0 pgmap v269: 137 pgs: 137 active+clean; 281 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 33 MiB/s rd, 8.3 MiB/s wr, 170 op/s 2024-03-07T03:12:26.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:25 smithi145 bash[21339]: cluster 2024-03-07T03:12:25.450388+0000 mon.a (mon.0) 957 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2024-03-07T03:12:26.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:25 smithi145 bash[30906]: cluster 2024-03-07T03:12:24.484380+0000 mgr.y (mgr.24302) 241 : cluster 0 pgmap v269: 137 pgs: 137 active+clean; 281 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 33 MiB/s rd, 8.3 MiB/s wr, 170 op/s 2024-03-07T03:12:26.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:25 smithi145 bash[30906]: cluster 2024-03-07T03:12:25.450388+0000 mon.a (mon.0) 957 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2024-03-07T03:12:26.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:25 smithi189 bash[23614]: cluster 2024-03-07T03:12:24.484380+0000 mgr.y (mgr.24302) 241 : cluster 0 pgmap v269: 137 pgs: 137 active+clean; 281 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 33 MiB/s rd, 8.3 MiB/s wr, 170 op/s 2024-03-07T03:12:26.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:25 smithi189 bash[23614]: cluster 2024-03-07T03:12:25.450388+0000 mon.a (mon.0) 957 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2024-03-07T03:12:27.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:26 smithi145 bash[21339]: audit 2024-03-07T03:12:26.223813+0000 mon.c (mon.2) 139 : audit 0 from='client.? 172.21.15.145:0/315359946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:27.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:26 smithi145 bash[30906]: audit 2024-03-07T03:12:26.223813+0000 mon.c (mon.2) 139 : audit 0 from='client.? 172.21.15.145:0/315359946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:27.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:26 smithi189 bash[23614]: audit 2024-03-07T03:12:26.223813+0000 mon.c (mon.2) 139 : audit 0 from='client.? 172.21.15.145:0/315359946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:28.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:27 smithi145 bash[21339]: cluster 2024-03-07T03:12:26.485147+0000 mgr.y (mgr.24302) 242 : cluster 0 pgmap v271: 137 pgs: 137 active+clean; 281 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 164 op/s 2024-03-07T03:12:28.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:27 smithi145 bash[21339]: cluster 2024-03-07T03:12:26.652336+0000 mon.a (mon.0) 958 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2024-03-07T03:12:28.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:27 smithi145 bash[30906]: cluster 2024-03-07T03:12:26.485147+0000 mgr.y (mgr.24302) 242 : cluster 0 pgmap v271: 137 pgs: 137 active+clean; 281 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 164 op/s 2024-03-07T03:12:28.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:27 smithi145 bash[30906]: cluster 2024-03-07T03:12:26.652336+0000 mon.a (mon.0) 958 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2024-03-07T03:12:28.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:27 smithi189 bash[23614]: cluster 2024-03-07T03:12:26.485147+0000 mgr.y (mgr.24302) 242 : cluster 0 pgmap v271: 137 pgs: 137 active+clean; 281 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 164 op/s 2024-03-07T03:12:28.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:27 smithi189 bash[23614]: cluster 2024-03-07T03:12:26.652336+0000 mon.a (mon.0) 958 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2024-03-07T03:12:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:28 smithi145 bash[21339]: cluster 2024-03-07T03:12:27.662698+0000 mon.a (mon.0) 959 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2024-03-07T03:12:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:28 smithi145 bash[21339]: cluster 2024-03-07T03:12:28.430996+0000 mon.a (mon.0) 960 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2024-03-07T03:12:29.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:28 smithi145 bash[30906]: cluster 2024-03-07T03:12:27.662698+0000 mon.a (mon.0) 959 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2024-03-07T03:12:29.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:28 smithi145 bash[30906]: cluster 2024-03-07T03:12:28.430996+0000 mon.a (mon.0) 960 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2024-03-07T03:12:29.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:28 smithi189 bash[23614]: cluster 2024-03-07T03:12:27.662698+0000 mon.a (mon.0) 959 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2024-03-07T03:12:29.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:28 smithi189 bash[23614]: cluster 2024-03-07T03:12:28.430996+0000 mon.a (mon.0) 960 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2024-03-07T03:12:29.189 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:12:29.471 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:12:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:29 smithi145 bash[21339]: cluster 2024-03-07T03:12:28.486008+0000 mgr.y (mgr.24302) 243 : cluster 0 pgmap v275: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 12 MiB/s wr, 124 op/s 2024-03-07T03:12:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:29 smithi145 bash[21339]: audit 2024-03-07T03:12:28.704032+0000 mon.c (mon.2) 140 : audit 0 from='client.? 172.21.15.145:0/1015559428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:30.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:29 smithi145 bash[30906]: cluster 2024-03-07T03:12:28.486008+0000 mgr.y (mgr.24302) 243 : cluster 0 pgmap v275: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 12 MiB/s wr, 124 op/s 2024-03-07T03:12:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:29 smithi145 bash[30906]: audit 2024-03-07T03:12:28.704032+0000 mon.c (mon.2) 140 : audit 0 from='client.? 172.21.15.145:0/1015559428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:30.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:29 smithi189 bash[23614]: cluster 2024-03-07T03:12:28.486008+0000 mgr.y (mgr.24302) 243 : cluster 0 pgmap v275: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 12 MiB/s wr, 124 op/s 2024-03-07T03:12:30.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:29 smithi189 bash[23614]: audit 2024-03-07T03:12:28.704032+0000 mon.c (mon.2) 140 : audit 0 from='client.? 172.21.15.145:0/1015559428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:31.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:31 smithi145 bash[21339]: cluster 2024-03-07T03:12:30.449560+0000 mon.a (mon.0) 961 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2024-03-07T03:12:31.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:31 smithi145 bash[21339]: cluster 2024-03-07T03:12:30.486738+0000 mgr.y (mgr.24302) 244 : cluster 0 pgmap v277: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 12 MiB/s wr, 191 op/s 2024-03-07T03:12:31.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:31 smithi145 bash[21339]: audit 2024-03-07T03:12:31.147867+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.145:0/2598811465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:31.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:31 smithi145 bash[30906]: cluster 2024-03-07T03:12:30.449560+0000 mon.a (mon.0) 961 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2024-03-07T03:12:31.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:31 smithi145 bash[30906]: cluster 2024-03-07T03:12:30.486738+0000 mgr.y (mgr.24302) 244 : cluster 0 pgmap v277: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 12 MiB/s wr, 191 op/s 2024-03-07T03:12:31.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:31 smithi145 bash[30906]: audit 2024-03-07T03:12:31.147867+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.145:0/2598811465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:31.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:31 smithi189 bash[23614]: cluster 2024-03-07T03:12:30.449560+0000 mon.a (mon.0) 961 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2024-03-07T03:12:31.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:31 smithi189 bash[23614]: cluster 2024-03-07T03:12:30.486738+0000 mgr.y (mgr.24302) 244 : cluster 0 pgmap v277: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 12 MiB/s wr, 191 op/s 2024-03-07T03:12:31.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:31 smithi189 bash[23614]: audit 2024-03-07T03:12:31.147867+0000 mon.c (mon.2) 141 : audit 0 from='client.? 172.21.15.145:0/2598811465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:32.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:32 smithi145 bash[21339]: cluster 2024-03-07T03:12:31.458273+0000 mon.a (mon.0) 962 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2024-03-07T03:12:32.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:32 smithi145 bash[30906]: cluster 2024-03-07T03:12:31.458273+0000 mon.a (mon.0) 962 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2024-03-07T03:12:32.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:12:32 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:12:32] "GET /metrics HTTP/1.1" 200 38625 "" "Prometheus/2.43.0" 2024-03-07T03:12:32.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:32 smithi189 bash[23614]: cluster 2024-03-07T03:12:31.458273+0000 mon.a (mon.0) 962 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2024-03-07T03:12:33.063 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.SnapsOtherPool (12915 ms) 2024-03-07T03:12:33.063 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.SnapsDataPool 2024-03-07T03:12:33.147 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:33.177 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:12:33.286 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:12:33.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:33 smithi145 bash[21339]: cluster 2024-03-07T03:12:32.459337+0000 mon.a (mon.0) 963 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2024-03-07T03:12:33.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:33 smithi145 bash[21339]: audit 2024-03-07T03:12:32.467009+0000 mon.c (mon.2) 142 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:33.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:33 smithi145 bash[21339]: cluster 2024-03-07T03:12:32.492428+0000 mgr.y (mgr.24302) 245 : cluster 0 pgmap v280: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 12 MiB/s wr, 188 op/s 2024-03-07T03:12:33.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:33 smithi145 bash[30906]: cluster 2024-03-07T03:12:32.459337+0000 mon.a (mon.0) 963 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2024-03-07T03:12:33.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:33 smithi145 bash[30906]: audit 2024-03-07T03:12:32.467009+0000 mon.c (mon.2) 142 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:33.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:33 smithi145 bash[30906]: cluster 2024-03-07T03:12:32.492428+0000 mgr.y (mgr.24302) 245 : cluster 0 pgmap v280: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 12 MiB/s wr, 188 op/s 2024-03-07T03:12:33.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:33 smithi189 bash[23614]: cluster 2024-03-07T03:12:32.459337+0000 mon.a (mon.0) 963 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2024-03-07T03:12:33.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:33 smithi189 bash[23614]: audit 2024-03-07T03:12:32.467009+0000 mon.c (mon.2) 142 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:33.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:33 smithi189 bash[23614]: cluster 2024-03-07T03:12:32.492428+0000 mgr.y (mgr.24302) 245 : cluster 0 pgmap v280: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 12 MiB/s wr, 188 op/s 2024-03-07T03:12:34.518 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 8388609~8388609 2024-03-07T03:12:34.550 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 0~1000 2024-03-07T03:12:34.550 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 1010~1000 2024-03-07T03:12:34.550 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:12:34.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:34 smithi145 bash[21339]: cluster 2024-03-07T03:12:33.480673+0000 mon.a (mon.0) 964 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2024-03-07T03:12:34.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:34 smithi145 bash[21339]: audit 2024-03-07T03:12:33.615122+0000 mon.a (mon.0) 965 : audit 0 from='client.? 172.21.15.145:0/2840291878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:34.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:34 smithi145 bash[30906]: cluster 2024-03-07T03:12:33.480673+0000 mon.a (mon.0) 964 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2024-03-07T03:12:34.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:34 smithi145 bash[30906]: audit 2024-03-07T03:12:33.615122+0000 mon.a (mon.0) 965 : audit 0 from='client.? 172.21.15.145:0/2840291878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:34.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:34 smithi189 bash[23614]: cluster 2024-03-07T03:12:33.480673+0000 mon.a (mon.0) 964 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2024-03-07T03:12:34.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:34 smithi189 bash[23614]: audit 2024-03-07T03:12:33.615122+0000 mon.a (mon.0) 965 : audit 0 from='client.? 172.21.15.145:0/2840291878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:35.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:35 smithi145 bash[30906]: cluster 2024-03-07T03:12:34.493281+0000 mgr.y (mgr.24302) 246 : cluster 0 pgmap v283: 137 pgs: 137 active+clean; 353 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 82 MiB/s rd, 14 MiB/s wr, 435 op/s 2024-03-07T03:12:35.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:35 smithi145 bash[30906]: cluster 2024-03-07T03:12:34.494830+0000 mon.a (mon.0) 966 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2024-03-07T03:12:35.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:35 smithi145 bash[21339]: cluster 2024-03-07T03:12:34.493281+0000 mgr.y (mgr.24302) 246 : cluster 0 pgmap v283: 137 pgs: 137 active+clean; 353 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 82 MiB/s rd, 14 MiB/s wr, 435 op/s 2024-03-07T03:12:35.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:35 smithi145 bash[21339]: cluster 2024-03-07T03:12:34.494830+0000 mon.a (mon.0) 966 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2024-03-07T03:12:35.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:35 smithi189 bash[23614]: cluster 2024-03-07T03:12:34.493281+0000 mgr.y (mgr.24302) 246 : cluster 0 pgmap v283: 137 pgs: 137 active+clean; 353 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 82 MiB/s rd, 14 MiB/s wr, 435 op/s 2024-03-07T03:12:35.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:35 smithi189 bash[23614]: cluster 2024-03-07T03:12:34.494830+0000 mon.a (mon.0) 966 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2024-03-07T03:12:36.535 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 8388609~1000 2024-03-07T03:12:36.551 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 16777228~1000 2024-03-07T03:12:36.551 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 167772160 2024-03-07T03:12:36.579 INFO:tasks.workunit.client.0.smithi145.stdout:write: 2 83886079~8388609 2024-03-07T03:12:36.626 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:12:36.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:36 smithi145 bash[21339]: cluster 2024-03-07T03:12:35.519216+0000 mon.a (mon.0) 967 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2024-03-07T03:12:36.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:36 smithi145 bash[21339]: audit 2024-03-07T03:12:36.052465+0000 mon.a (mon.0) 968 : audit 0 from='client.? 172.21.15.145:0/2850733097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:36.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:36 smithi145 bash[30906]: cluster 2024-03-07T03:12:35.519216+0000 mon.a (mon.0) 967 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2024-03-07T03:12:36.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:36 smithi145 bash[30906]: audit 2024-03-07T03:12:36.052465+0000 mon.a (mon.0) 968 : audit 0 from='client.? 172.21.15.145:0/2850733097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:36 smithi189 bash[23614]: cluster 2024-03-07T03:12:35.519216+0000 mon.a (mon.0) 967 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2024-03-07T03:12:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:36 smithi189 bash[23614]: audit 2024-03-07T03:12:36.052465+0000 mon.a (mon.0) 968 : audit 0 from='client.? 172.21.15.145:0/2850733097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:37.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:37 smithi145 bash[21339]: cluster 2024-03-07T03:12:36.494162+0000 mgr.y (mgr.24302) 247 : cluster 0 pgmap v285: 137 pgs: 137 active+clean; 353 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 82 MiB/s rd, 14 MiB/s wr, 431 op/s 2024-03-07T03:12:37.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:37 smithi145 bash[21339]: cluster 2024-03-07T03:12:36.526791+0000 mon.a (mon.0) 969 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2024-03-07T03:12:37.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:37 smithi145 bash[30906]: cluster 2024-03-07T03:12:36.494162+0000 mgr.y (mgr.24302) 247 : cluster 0 pgmap v285: 137 pgs: 137 active+clean; 353 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 82 MiB/s rd, 14 MiB/s wr, 431 op/s 2024-03-07T03:12:37.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:37 smithi145 bash[30906]: cluster 2024-03-07T03:12:36.526791+0000 mon.a (mon.0) 969 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2024-03-07T03:12:37.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:37 smithi189 bash[23614]: cluster 2024-03-07T03:12:36.494162+0000 mgr.y (mgr.24302) 247 : cluster 0 pgmap v285: 137 pgs: 137 active+clean; 353 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 82 MiB/s rd, 14 MiB/s wr, 431 op/s 2024-03-07T03:12:37.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:37 smithi189 bash[23614]: cluster 2024-03-07T03:12:36.526791+0000 mon.a (mon.0) 969 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2024-03-07T03:12:38.591 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:38.642 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 83886079~1 2024-03-07T03:12:38.664 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:12:38.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:38 smithi145 bash[21339]: cluster 2024-03-07T03:12:37.534340+0000 mon.a (mon.0) 970 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2024-03-07T03:12:38.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:38 smithi145 bash[21339]: cluster 2024-03-07T03:12:38.434006+0000 mon.a (mon.0) 971 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2024-03-07T03:12:38.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:38 smithi145 bash[21339]: audit 2024-03-07T03:12:38.492166+0000 mon.c (mon.2) 143 : audit 0 from='client.? 172.21.15.145:0/3978998571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:38.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:38 smithi145 bash[30906]: cluster 2024-03-07T03:12:37.534340+0000 mon.a (mon.0) 970 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2024-03-07T03:12:38.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:38 smithi145 bash[30906]: cluster 2024-03-07T03:12:38.434006+0000 mon.a (mon.0) 971 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2024-03-07T03:12:38.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:38 smithi145 bash[30906]: audit 2024-03-07T03:12:38.492166+0000 mon.c (mon.2) 143 : audit 0 from='client.? 172.21.15.145:0/3978998571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:38.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:38 smithi189 bash[23614]: cluster 2024-03-07T03:12:37.534340+0000 mon.a (mon.0) 970 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2024-03-07T03:12:38.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:38 smithi189 bash[23614]: cluster 2024-03-07T03:12:38.434006+0000 mon.a (mon.0) 971 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2024-03-07T03:12:38.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:38 smithi189 bash[23614]: audit 2024-03-07T03:12:38.492166+0000 mon.c (mon.2) 143 : audit 0 from='client.? 172.21.15.145:0/3978998571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:39.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:39 smithi145 bash[21339]: cluster 2024-03-07T03:12:38.495574+0000 mgr.y (mgr.24302) 248 : cluster 0 pgmap v289: 137 pgs: 137 active+clean; 385 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 6.0 MiB/s wr, 53 op/s 2024-03-07T03:12:39.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:39 smithi145 bash[30906]: cluster 2024-03-07T03:12:38.495574+0000 mgr.y (mgr.24302) 248 : cluster 0 pgmap v289: 137 pgs: 137 active+clean; 385 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 6.0 MiB/s wr, 53 op/s 2024-03-07T03:12:39.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:39 smithi189 bash[23614]: cluster 2024-03-07T03:12:38.495574+0000 mgr.y (mgr.24302) 248 : cluster 0 pgmap v289: 137 pgs: 137 active+clean; 385 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 6.0 MiB/s wr, 53 op/s 2024-03-07T03:12:40.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:40 smithi145 bash[30906]: cluster 2024-03-07T03:12:39.545047+0000 mon.a (mon.0) 972 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2024-03-07T03:12:40.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:40 smithi145 bash[21339]: cluster 2024-03-07T03:12:39.545047+0000 mon.a (mon.0) 972 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2024-03-07T03:12:40.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:40 smithi189 bash[23614]: cluster 2024-03-07T03:12:39.545047+0000 mon.a (mon.0) 972 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2024-03-07T03:12:41.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:41 smithi145 bash[21339]: cluster 2024-03-07T03:12:40.496343+0000 mgr.y (mgr.24302) 249 : cluster 0 pgmap v291: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-03-07T03:12:41.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:41 smithi145 bash[21339]: cluster 2024-03-07T03:12:40.555171+0000 mon.a (mon.0) 973 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2024-03-07T03:12:41.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:41 smithi145 bash[21339]: audit 2024-03-07T03:12:40.936524+0000 mon.a (mon.0) 974 : audit 0 from='client.? 172.21.15.145:0/2647366014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:41.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:41 smithi145 bash[30906]: cluster 2024-03-07T03:12:40.496343+0000 mgr.y (mgr.24302) 249 : cluster 0 pgmap v291: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-03-07T03:12:41.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:41 smithi145 bash[30906]: cluster 2024-03-07T03:12:40.555171+0000 mon.a (mon.0) 973 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2024-03-07T03:12:41.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:41 smithi145 bash[30906]: audit 2024-03-07T03:12:40.936524+0000 mon.a (mon.0) 974 : audit 0 from='client.? 172.21.15.145:0/2647366014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:41.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:41 smithi189 bash[23614]: cluster 2024-03-07T03:12:40.496343+0000 mgr.y (mgr.24302) 249 : cluster 0 pgmap v291: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-03-07T03:12:41.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:41 smithi189 bash[23614]: cluster 2024-03-07T03:12:40.555171+0000 mon.a (mon.0) 973 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2024-03-07T03:12:41.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:41 smithi189 bash[23614]: audit 2024-03-07T03:12:40.936524+0000 mon.a (mon.0) 974 : audit 0 from='client.? 172.21.15.145:0/2647366014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:42.460 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:12:42.712 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:12:42.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:42 smithi145 bash[21339]: cluster 2024-03-07T03:12:41.555731+0000 mon.a (mon.0) 975 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2024-03-07T03:12:42.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:12:42 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:12:42] "GET /metrics HTTP/1.1" 200 38627 "" "Prometheus/2.43.0" 2024-03-07T03:12:42.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:42 smithi145 bash[30906]: cluster 2024-03-07T03:12:41.555731+0000 mon.a (mon.0) 975 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2024-03-07T03:12:42.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:42 smithi189 bash[23614]: cluster 2024-03-07T03:12:41.555731+0000 mon.a (mon.0) 975 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2024-03-07T03:12:43.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:43 smithi189 bash[23614]: cluster 2024-03-07T03:12:42.497088+0000 mgr.y (mgr.24302) 250 : cluster 0 pgmap v294: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 7.9 MiB/s wr, 102 op/s 2024-03-07T03:12:43.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:43 smithi189 bash[23614]: audit 2024-03-07T03:12:43.371388+0000 mon.c (mon.2) 144 : audit 0 from='client.? 172.21.15.145:0/4179058660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:43.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:43 smithi189 bash[23614]: audit 2024-03-07T03:12:43.446536+0000 mon.a (mon.0) 976 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:43.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:43 smithi189 bash[23614]: audit 2024-03-07T03:12:43.448045+0000 mon.c (mon.2) 145 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:12:43.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:43 smithi189 bash[23614]: audit 2024-03-07T03:12:43.448697+0000 mon.a (mon.0) 977 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:12:43.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:43 smithi189 bash[23614]: audit 2024-03-07T03:12:43.450344+0000 mon.c (mon.2) 146 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:12:43.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:43 smithi189 bash[23614]: audit 2024-03-07T03:12:43.451978+0000 mon.c (mon.2) 147 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:12:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[21339]: cluster 2024-03-07T03:12:42.497088+0000 mgr.y (mgr.24302) 250 : cluster 0 pgmap v294: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 7.9 MiB/s wr, 102 op/s 2024-03-07T03:12:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[21339]: audit 2024-03-07T03:12:43.371388+0000 mon.c (mon.2) 144 : audit 0 from='client.? 172.21.15.145:0/4179058660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[21339]: audit 2024-03-07T03:12:43.446536+0000 mon.a (mon.0) 976 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[21339]: audit 2024-03-07T03:12:43.448045+0000 mon.c (mon.2) 145 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:12:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[21339]: audit 2024-03-07T03:12:43.448697+0000 mon.a (mon.0) 977 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:12:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[21339]: audit 2024-03-07T03:12:43.450344+0000 mon.c (mon.2) 146 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:12:44.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[21339]: audit 2024-03-07T03:12:43.451978+0000 mon.c (mon.2) 147 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:12:44.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[30906]: cluster 2024-03-07T03:12:42.497088+0000 mgr.y (mgr.24302) 250 : cluster 0 pgmap v294: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 7.9 MiB/s wr, 102 op/s 2024-03-07T03:12:44.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[30906]: audit 2024-03-07T03:12:43.371388+0000 mon.c (mon.2) 144 : audit 0 from='client.? 172.21.15.145:0/4179058660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:44.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[30906]: audit 2024-03-07T03:12:43.446536+0000 mon.a (mon.0) 976 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:44.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[30906]: audit 2024-03-07T03:12:43.448045+0000 mon.c (mon.2) 145 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:12:44.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[30906]: audit 2024-03-07T03:12:43.448697+0000 mon.a (mon.0) 977 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:12:44.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[30906]: audit 2024-03-07T03:12:43.450344+0000 mon.c (mon.2) 146 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:12:44.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:43 smithi145 bash[30906]: audit 2024-03-07T03:12:43.451978+0000 mon.c (mon.2) 147 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:12:44.802 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:44 smithi189 bash[23614]: cephadm 2024-03-07T03:12:43.438448+0000 mgr.y (mgr.24302) 251 : cephadm 1 Upgrade: Updating mgr.x 2024-03-07T03:12:44.802 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:44 smithi189 bash[23614]: cephadm 2024-03-07T03:12:43.453517+0000 mgr.y (mgr.24302) 252 : cephadm 1 Deploying daemon mgr.x on smithi189 2024-03-07T03:12:44.802 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:44 smithi189 bash[23614]: cluster 2024-03-07T03:12:43.579496+0000 mon.a (mon.0) 978 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2024-03-07T03:12:45.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:44 smithi145 bash[21339]: cephadm 2024-03-07T03:12:43.438448+0000 mgr.y (mgr.24302) 251 : cephadm 1 Upgrade: Updating mgr.x 2024-03-07T03:12:45.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:44 smithi145 bash[21339]: cephadm 2024-03-07T03:12:43.453517+0000 mgr.y (mgr.24302) 252 : cephadm 1 Deploying daemon mgr.x on smithi189 2024-03-07T03:12:45.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:44 smithi145 bash[21339]: cluster 2024-03-07T03:12:43.579496+0000 mon.a (mon.0) 978 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2024-03-07T03:12:45.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:44 smithi145 bash[30906]: cephadm 2024-03-07T03:12:43.438448+0000 mgr.y (mgr.24302) 251 : cephadm 1 Upgrade: Updating mgr.x 2024-03-07T03:12:45.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:44 smithi145 bash[30906]: cephadm 2024-03-07T03:12:43.453517+0000 mgr.y (mgr.24302) 252 : cephadm 1 Deploying daemon mgr.x on smithi189 2024-03-07T03:12:45.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:44 smithi145 bash[30906]: cluster 2024-03-07T03:12:43.579496+0000 mon.a (mon.0) 978 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2024-03-07T03:12:45.612 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:12:45 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:45.613 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:12:45 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:45.613 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:12:45 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:45.613 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:12:45 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:45.614 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:45 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:45.614 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:45 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:45.614 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:45 smithi189 bash[23614]: cluster 2024-03-07T03:12:44.498164+0000 mgr.y (mgr.24302) 253 : cluster 0 pgmap v296: 137 pgs: 137 active+clean; 421 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 4.9 MiB/s wr, 394 op/s 2024-03-07T03:12:45.614 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:45 smithi189 bash[23614]: cluster 2024-03-07T03:12:44.599967+0000 mon.a (mon.0) 979 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2024-03-07T03:12:45.614 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:12:45 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:45.615 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:12:45 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:45.615 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:12:45 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:45.967 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:45 smithi189 systemd[1]: Stopping Ceph mgr.x for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:12:46.054 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.SnapsDataPool (12991 ms) 2024-03-07T03:12:46.054 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.SnapsShrinkAfterPrepare 2024-03-07T03:12:46.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:45 smithi145 bash[21339]: cluster 2024-03-07T03:12:44.498164+0000 mgr.y (mgr.24302) 253 : cluster 0 pgmap v296: 137 pgs: 137 active+clean; 421 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 4.9 MiB/s wr, 394 op/s 2024-03-07T03:12:46.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:45 smithi145 bash[21339]: cluster 2024-03-07T03:12:44.599967+0000 mon.a (mon.0) 979 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2024-03-07T03:12:46.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:45 smithi145 bash[30906]: cluster 2024-03-07T03:12:44.498164+0000 mgr.y (mgr.24302) 253 : cluster 0 pgmap v296: 137 pgs: 137 active+clean; 421 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 4.9 MiB/s wr, 394 op/s 2024-03-07T03:12:46.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:45 smithi145 bash[30906]: cluster 2024-03-07T03:12:44.599967+0000 mon.a (mon.0) 979 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2024-03-07T03:12:46.137 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:46.169 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:12:46.286 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:12:46.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:46 smithi189 bash[51985]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mgr-x 2024-03-07T03:12:46.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:46 smithi189 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-03-07T03:12:46.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:46 smithi189 bash[52047]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mgr-x 2024-03-07T03:12:46.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:46 smithi189 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.x.service: Failed with result 'exit-code'. 2024-03-07T03:12:46.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:46 smithi189 systemd[1]: Stopped Ceph mgr.x for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:12:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:46 smithi145 bash[21339]: cluster 2024-03-07T03:12:45.608866+0000 mon.a (mon.0) 980 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2024-03-07T03:12:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:46 smithi145 bash[21339]: audit 2024-03-07T03:12:45.838403+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.145:0/1495034921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:47.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:46 smithi145 bash[30906]: cluster 2024-03-07T03:12:45.608866+0000 mon.a (mon.0) 980 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2024-03-07T03:12:47.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:46 smithi145 bash[30906]: audit 2024-03-07T03:12:45.838403+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.145:0/1495034921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:47.080 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:12:46 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:47.081 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:12:46 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:47.081 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:12:46 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:47.081 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:12:46 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:47.082 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:12:46 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:47.082 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:46 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:47.082 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:46 smithi189 systemd[1]: Started Ceph mgr.x for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:12:47.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:46 smithi189 bash[23614]: cluster 2024-03-07T03:12:45.608866+0000 mon.a (mon.0) 980 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2024-03-07T03:12:47.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:46 smithi189 bash[23614]: audit 2024-03-07T03:12:45.838403+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.145:0/1495034921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:47.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:46 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:47.083 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:12:46 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:47.083 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:12:46 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:12:47.647 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 8388609~8388609 2024-03-07T03:12:47.679 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 0~1000 2024-03-07T03:12:47.680 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 1010~1000 2024-03-07T03:12:47.680 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:12:47.941 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:47 smithi189 bash[52103]: debug 2024-03-07T03:12:47.804+0000 7f975a02c700 1 -- 172.21.15.189:0/2329127734 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563a73a025a0 con 0x563a74786400 2024-03-07T03:12:47.942 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:47 smithi189 bash[23614]: cluster 2024-03-07T03:12:46.498858+0000 mgr.y (mgr.24302) 254 : cluster 0 pgmap v299: 137 pgs: 137 active+clean; 421 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 4.9 MiB/s wr, 395 op/s 2024-03-07T03:12:47.942 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:47 smithi189 bash[23614]: cluster 2024-03-07T03:12:46.630817+0000 mon.a (mon.0) 982 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2024-03-07T03:12:47.942 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:47 smithi189 bash[23614]: audit 2024-03-07T03:12:46.825574+0000 mon.a (mon.0) 983 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:47.942 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:47 smithi189 bash[23614]: audit 2024-03-07T03:12:46.843379+0000 mon.a (mon.0) 984 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:47.942 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:47 smithi189 bash[23614]: audit 2024-03-07T03:12:46.877424+0000 mon.c (mon.2) 148 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:12:47.942 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:47 smithi189 bash[23614]: audit 2024-03-07T03:12:47.455627+0000 mon.c (mon.2) 149 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:48.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:47 smithi145 bash[30906]: cluster 2024-03-07T03:12:46.498858+0000 mgr.y (mgr.24302) 254 : cluster 0 pgmap v299: 137 pgs: 137 active+clean; 421 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 4.9 MiB/s wr, 395 op/s 2024-03-07T03:12:48.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:47 smithi145 bash[30906]: cluster 2024-03-07T03:12:46.630817+0000 mon.a (mon.0) 982 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2024-03-07T03:12:48.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:47 smithi145 bash[30906]: audit 2024-03-07T03:12:46.825574+0000 mon.a (mon.0) 983 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:48.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:47 smithi145 bash[30906]: audit 2024-03-07T03:12:46.843379+0000 mon.a (mon.0) 984 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:48.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:47 smithi145 bash[30906]: audit 2024-03-07T03:12:46.877424+0000 mon.c (mon.2) 148 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:12:48.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:47 smithi145 bash[30906]: audit 2024-03-07T03:12:47.455627+0000 mon.c (mon.2) 149 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:47 smithi145 bash[21339]: cluster 2024-03-07T03:12:46.498858+0000 mgr.y (mgr.24302) 254 : cluster 0 pgmap v299: 137 pgs: 137 active+clean; 421 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 4.9 MiB/s wr, 395 op/s 2024-03-07T03:12:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:47 smithi145 bash[21339]: cluster 2024-03-07T03:12:46.630817+0000 mon.a (mon.0) 982 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2024-03-07T03:12:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:47 smithi145 bash[21339]: audit 2024-03-07T03:12:46.825574+0000 mon.a (mon.0) 983 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:47 smithi145 bash[21339]: audit 2024-03-07T03:12:46.843379+0000 mon.a (mon.0) 984 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:47 smithi145 bash[21339]: audit 2024-03-07T03:12:46.877424+0000 mon.c (mon.2) 148 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:12:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:47 smithi145 bash[21339]: audit 2024-03-07T03:12:47.455627+0000 mon.c (mon.2) 149 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:48.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:47 smithi189 bash[52103]: debug 2024-03-07T03:12:47.948+0000 7f976422e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:12:49.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:48 smithi145 bash[21339]: cluster 2024-03-07T03:12:47.629033+0000 mon.a (mon.0) 985 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2024-03-07T03:12:49.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:48 smithi145 bash[21339]: audit 2024-03-07T03:12:48.278032+0000 mon.a (mon.0) 986 : audit 0 from='client.? 172.21.15.145:0/278159045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:49.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:48 smithi145 bash[30906]: cluster 2024-03-07T03:12:47.629033+0000 mon.a (mon.0) 985 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2024-03-07T03:12:49.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:48 smithi145 bash[30906]: audit 2024-03-07T03:12:48.278032+0000 mon.a (mon.0) 986 : audit 0 from='client.? 172.21.15.145:0/278159045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:49.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:48 smithi189 bash[23614]: cluster 2024-03-07T03:12:47.629033+0000 mon.a (mon.0) 985 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2024-03-07T03:12:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:48 smithi189 bash[23614]: audit 2024-03-07T03:12:48.278032+0000 mon.a (mon.0) 986 : audit 0 from='client.? 172.21.15.145:0/278159045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:49.685 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 8388609~1000 2024-03-07T03:12:49.695 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 16777228~1000 2024-03-07T03:12:49.695 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 167772160 2024-03-07T03:12:49.707 INFO:tasks.workunit.client.0.smithi145.stdout:write: 2 83886079~8388609 2024-03-07T03:12:49.730 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:12:49.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:49 smithi189 bash[23614]: cluster 2024-03-07T03:12:48.500242+0000 mgr.y (mgr.24302) 255 : cluster 0 pgmap v302: 137 pgs: 137 active+clean; 429 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 73 op/s 2024-03-07T03:12:49.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:49 smithi189 bash[23614]: cluster 2024-03-07T03:12:48.657303+0000 mon.a (mon.0) 987 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2024-03-07T03:12:49.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:49 smithi189 bash[52103]: debug 2024-03-07T03:12:49.564+0000 7f976422e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:12:49.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:49 smithi189 bash[52103]: debug 2024-03-07T03:12:49.664+0000 7f976422e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:12:50.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:49 smithi145 bash[21339]: cluster 2024-03-07T03:12:48.500242+0000 mgr.y (mgr.24302) 255 : cluster 0 pgmap v302: 137 pgs: 137 active+clean; 429 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 73 op/s 2024-03-07T03:12:50.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:49 smithi145 bash[21339]: cluster 2024-03-07T03:12:48.657303+0000 mon.a (mon.0) 987 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2024-03-07T03:12:50.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:49 smithi145 bash[30906]: cluster 2024-03-07T03:12:48.500242+0000 mgr.y (mgr.24302) 255 : cluster 0 pgmap v302: 137 pgs: 137 active+clean; 429 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 73 op/s 2024-03-07T03:12:50.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:49 smithi145 bash[30906]: cluster 2024-03-07T03:12:48.657303+0000 mon.a (mon.0) 987 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2024-03-07T03:12:50.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:49 smithi189 bash[52103]: debug 2024-03-07T03:12:49.928+0000 7f976422e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:12:50.961 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:50 smithi189 bash[52103]: debug 2024-03-07T03:12:50.752+0000 7f976422e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:12:50.961 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:50 smithi189 bash[52103]: debug 2024-03-07T03:12:50.864+0000 7f976422e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:12:50.962 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:50 smithi189 bash[23614]: cluster 2024-03-07T03:12:49.667533+0000 mon.a (mon.0) 988 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2024-03-07T03:12:51.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:50 smithi145 bash[21339]: cluster 2024-03-07T03:12:49.667533+0000 mon.a (mon.0) 988 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2024-03-07T03:12:51.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:50 smithi145 bash[30906]: cluster 2024-03-07T03:12:49.667533+0000 mon.a (mon.0) 988 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2024-03-07T03:12:51.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:50 smithi189 bash[52103]: debug 2024-03-07T03:12:50.968+0000 7f976422e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:12:51.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:51 smithi189 bash[52103]: debug 2024-03-07T03:12:51.328+0000 7f976422e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:12:51.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:51 smithi189 bash[52103]: debug 2024-03-07T03:12:51.440+0000 7f976422e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:12:51.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:51 smithi189 bash[52103]: debug 2024-03-07T03:12:51.544+0000 7f976422e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:12:51.698 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:51.745 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 83886079~1 2024-03-07T03:12:51.757 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:12:51.932 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:51 smithi189 bash[23614]: cluster 2024-03-07T03:12:50.500902+0000 mgr.y (mgr.24302) 256 : cluster 0 pgmap v305: 137 pgs: 137 active+clean; 425 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 140 op/s 2024-03-07T03:12:51.932 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:51 smithi189 bash[23614]: cluster 2024-03-07T03:12:50.676662+0000 mon.a (mon.0) 989 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2024-03-07T03:12:51.932 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:51 smithi189 bash[23614]: audit 2024-03-07T03:12:50.727708+0000 mon.a (mon.0) 990 : audit 0 from='client.? 172.21.15.145:0/1580569105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:52.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:51 smithi145 bash[21339]: cluster 2024-03-07T03:12:50.500902+0000 mgr.y (mgr.24302) 256 : cluster 0 pgmap v305: 137 pgs: 137 active+clean; 425 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 140 op/s 2024-03-07T03:12:52.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:51 smithi145 bash[21339]: cluster 2024-03-07T03:12:50.676662+0000 mon.a (mon.0) 989 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2024-03-07T03:12:52.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:51 smithi145 bash[21339]: audit 2024-03-07T03:12:50.727708+0000 mon.a (mon.0) 990 : audit 0 from='client.? 172.21.15.145:0/1580569105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:52.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:51 smithi145 bash[30906]: cluster 2024-03-07T03:12:50.500902+0000 mgr.y (mgr.24302) 256 : cluster 0 pgmap v305: 137 pgs: 137 active+clean; 425 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 140 op/s 2024-03-07T03:12:52.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:51 smithi145 bash[30906]: cluster 2024-03-07T03:12:50.676662+0000 mon.a (mon.0) 989 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2024-03-07T03:12:52.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:51 smithi145 bash[30906]: audit 2024-03-07T03:12:50.727708+0000 mon.a (mon.0) 990 : audit 0 from='client.? 172.21.15.145:0/1580569105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:52.201 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:51 smithi189 bash[52103]: debug 2024-03-07T03:12:51.940+0000 7f976422e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:12:52.492 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:52 smithi189 bash[52103]: debug 2024-03-07T03:12:52.208+0000 7f976422e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:12:52.684 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:12:52 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:12:52] "GET /metrics HTTP/1.1" 200 38623 "" "Prometheus/2.43.0" 2024-03-07T03:12:52.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:52 smithi189 bash[23614]: cluster 2024-03-07T03:12:51.676894+0000 mon.a (mon.0) 991 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2024-03-07T03:12:52.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:52 smithi189 bash[23614]: audit 2024-03-07T03:12:52.370437+0000 mon.a (mon.0) 992 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:52.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:52 smithi189 bash[23614]: audit 2024-03-07T03:12:52.381741+0000 mon.a (mon.0) 993 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:52.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:52 smithi189 bash[52103]: debug 2024-03-07T03:12:52.500+0000 7f976422e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:12:52.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:52 smithi189 bash[52103]: debug 2024-03-07T03:12:52.624+0000 7f976422e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:12:53.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:52 smithi145 bash[30906]: cluster 2024-03-07T03:12:51.676894+0000 mon.a (mon.0) 991 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2024-03-07T03:12:53.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:52 smithi145 bash[30906]: audit 2024-03-07T03:12:52.370437+0000 mon.a (mon.0) 992 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:52 smithi145 bash[30906]: audit 2024-03-07T03:12:52.381741+0000 mon.a (mon.0) 993 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:52 smithi145 bash[21339]: cluster 2024-03-07T03:12:51.676894+0000 mon.a (mon.0) 991 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2024-03-07T03:12:53.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:52 smithi145 bash[21339]: audit 2024-03-07T03:12:52.370437+0000 mon.a (mon.0) 992 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:52 smithi145 bash[21339]: audit 2024-03-07T03:12:52.381741+0000 mon.a (mon.0) 993 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.159 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:52 smithi189 bash[52103]: debug 2024-03-07T03:12:52.912+0000 7f976422e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:12:53.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:53 smithi189 bash[52103]: debug 2024-03-07T03:12:53.168+0000 7f976422e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:12:53.810 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[57902]: ts=2024-03-07T03:12:53.516Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:12:53.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[21339]: cluster 2024-03-07T03:12:52.501640+0000 mgr.y (mgr.24302) 257 : cluster 0 pgmap v308: 137 pgs: 137 active+clean; 425 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 8.0 KiB/s wr, 66 op/s 2024-03-07T03:12:53.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[21339]: cluster 2024-03-07T03:12:52.691880+0000 mon.a (mon.0) 994 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2024-03-07T03:12:53.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[21339]: audit 2024-03-07T03:12:52.790164+0000 mon.a (mon.0) 995 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[21339]: audit 2024-03-07T03:12:52.808274+0000 mon.a (mon.0) 996 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[21339]: audit 2024-03-07T03:12:53.148931+0000 mon.c (mon.2) 150 : audit 0 from='client.? 172.21.15.145:0/2100406390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:53.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[21339]: cluster 2024-03-07T03:12:53.441503+0000 mon.a (mon.0) 997 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2024-03-07T03:12:53.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[21339]: audit 2024-03-07T03:12:53.609484+0000 mon.a (mon.0) 998 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[21339]: audit 2024-03-07T03:12:53.625334+0000 mon.a (mon.0) 999 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[30906]: cluster 2024-03-07T03:12:52.501640+0000 mgr.y (mgr.24302) 257 : cluster 0 pgmap v308: 137 pgs: 137 active+clean; 425 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 8.0 KiB/s wr, 66 op/s 2024-03-07T03:12:53.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[30906]: cluster 2024-03-07T03:12:52.691880+0000 mon.a (mon.0) 994 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2024-03-07T03:12:53.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[30906]: audit 2024-03-07T03:12:52.790164+0000 mon.a (mon.0) 995 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[30906]: audit 2024-03-07T03:12:52.808274+0000 mon.a (mon.0) 996 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[30906]: audit 2024-03-07T03:12:53.148931+0000 mon.c (mon.2) 150 : audit 0 from='client.? 172.21.15.145:0/2100406390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:53.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[30906]: cluster 2024-03-07T03:12:53.441503+0000 mon.a (mon.0) 997 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2024-03-07T03:12:53.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[30906]: audit 2024-03-07T03:12:53.609484+0000 mon.a (mon.0) 998 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:53 smithi145 bash[30906]: audit 2024-03-07T03:12:53.625334+0000 mon.a (mon.0) 999 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.978 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:53 smithi189 bash[52103]: debug 2024-03-07T03:12:53.604+0000 7f976422e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:12:53.978 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:53 smithi189 bash[52103]: debug 2024-03-07T03:12:53.728+0000 7f976422e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:12:53.978 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:53 smithi189 bash[23614]: cluster 2024-03-07T03:12:52.501640+0000 mgr.y (mgr.24302) 257 : cluster 0 pgmap v308: 137 pgs: 137 active+clean; 425 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 8.0 KiB/s wr, 66 op/s 2024-03-07T03:12:53.978 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:53 smithi189 bash[23614]: cluster 2024-03-07T03:12:52.691880+0000 mon.a (mon.0) 994 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2024-03-07T03:12:53.978 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:53 smithi189 bash[23614]: audit 2024-03-07T03:12:52.790164+0000 mon.a (mon.0) 995 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.978 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:53 smithi189 bash[23614]: audit 2024-03-07T03:12:52.808274+0000 mon.a (mon.0) 996 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.978 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:53 smithi189 bash[23614]: audit 2024-03-07T03:12:53.148931+0000 mon.c (mon.2) 150 : audit 0 from='client.? 172.21.15.145:0/2100406390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:53.978 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:53 smithi189 bash[23614]: cluster 2024-03-07T03:12:53.441503+0000 mon.a (mon.0) 997 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2024-03-07T03:12:53.978 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:53 smithi189 bash[23614]: audit 2024-03-07T03:12:53.609484+0000 mon.a (mon.0) 998 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:53.978 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:53 smithi189 bash[23614]: audit 2024-03-07T03:12:53.625334+0000 mon.a (mon.0) 999 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:12:55.292 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 41943040 2024-03-07T03:12:55.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:53 smithi189 bash[52103]: debug 2024-03-07T03:12:53.984+0000 7f976422e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:12:55.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:55 smithi189 bash[52103]: debug 2024-03-07T03:12:55.028+0000 7f976422e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:12:55.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:55 smithi189 bash[52103]: debug 2024-03-07T03:12:55.148+0000 7f976422e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:12:55.331 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:12:55.510 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:12:55.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:55 smithi145 bash[21339]: cluster 2024-03-07T03:12:54.457415+0000 mon.a (mon.0) 1000 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2024-03-07T03:12:55.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:55 smithi145 bash[21339]: cluster 2024-03-07T03:12:54.502380+0000 mgr.y (mgr.24302) 258 : cluster 0 pgmap v312: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 4.2 MiB/s wr, 180 op/s 2024-03-07T03:12:55.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:55 smithi145 bash[30906]: cluster 2024-03-07T03:12:54.457415+0000 mon.a (mon.0) 1000 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2024-03-07T03:12:55.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:55 smithi145 bash[30906]: cluster 2024-03-07T03:12:54.502380+0000 mgr.y (mgr.24302) 258 : cluster 0 pgmap v312: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 4.2 MiB/s wr, 180 op/s 2024-03-07T03:12:55.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:55 smithi189 bash[23614]: cluster 2024-03-07T03:12:54.457415+0000 mon.a (mon.0) 1000 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2024-03-07T03:12:55.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:55 smithi189 bash[23614]: cluster 2024-03-07T03:12:54.502380+0000 mgr.y (mgr.24302) 258 : cluster 0 pgmap v312: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 4.2 MiB/s wr, 180 op/s 2024-03-07T03:12:55.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:55 smithi189 bash[52103]: debug 2024-03-07T03:12:55.496+0000 7f976422e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:12:55.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:55 smithi189 bash[52103]: debug 2024-03-07T03:12:55.624+0000 7f976422e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:12:56.334 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[52103]: debug 2024-03-07T03:12:56.088+0000 7f976422e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:12:56.334 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[52103]: debug 2024-03-07T03:12:56.212+0000 7f976422e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:12:56.334 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[52103]: [07/Mar/2024:03:12:56] ENGINE Bus STARTING 2024-03-07T03:12:56.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[21339]: audit 2024-03-07T03:12:55.643811+0000 mon.a (mon.0) 1001 : audit 0 from='client.? 172.21.15.145:0/1069444538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:56.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[21339]: cluster 2024-03-07T03:12:56.238109+0000 mon.a (mon.0) 1002 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:12:56.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[21339]: cluster 2024-03-07T03:12:56.238519+0000 mon.a (mon.0) 1003 : cluster 0 Standby manager daemon x started 2024-03-07T03:12:56.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[21339]: audit 2024-03-07T03:12:56.238554+0000 mon.b (mon.1) 31 : audit 0 from='mgr.? 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:12:56.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[21339]: audit 2024-03-07T03:12:56.239552+0000 mon.b (mon.1) 32 : audit 0 from='mgr.? 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:12:56.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[21339]: audit 2024-03-07T03:12:56.242705+0000 mon.b (mon.1) 33 : audit 0 from='mgr.? 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:12:56.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[21339]: audit 2024-03-07T03:12:56.244447+0000 mon.b (mon.1) 34 : audit 0 from='mgr.? 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:12:56.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[30906]: audit 2024-03-07T03:12:55.643811+0000 mon.a (mon.0) 1001 : audit 0 from='client.? 172.21.15.145:0/1069444538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:56.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[30906]: cluster 2024-03-07T03:12:56.238109+0000 mon.a (mon.0) 1002 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:12:56.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[30906]: cluster 2024-03-07T03:12:56.238519+0000 mon.a (mon.0) 1003 : cluster 0 Standby manager daemon x started 2024-03-07T03:12:56.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[30906]: audit 2024-03-07T03:12:56.238554+0000 mon.b (mon.1) 31 : audit 0 from='mgr.? 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:12:56.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[30906]: audit 2024-03-07T03:12:56.239552+0000 mon.b (mon.1) 32 : audit 0 from='mgr.? 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:12:56.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[30906]: audit 2024-03-07T03:12:56.242705+0000 mon.b (mon.1) 33 : audit 0 from='mgr.? 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:12:56.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:56 smithi145 bash[30906]: audit 2024-03-07T03:12:56.244447+0000 mon.b (mon.1) 34 : audit 0 from='mgr.? 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:12:56.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[52103]: CherryPy Checker: 2024-03-07T03:12:56.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[52103]: The Application mounted at '' has an empty config. 2024-03-07T03:12:56.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[52103]: [07/Mar/2024:03:12:56] ENGINE Serving on http://:::9283 2024-03-07T03:12:56.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[52103]: [07/Mar/2024:03:12:56] ENGINE Bus STARTED 2024-03-07T03:12:56.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[23614]: audit 2024-03-07T03:12:55.643811+0000 mon.a (mon.0) 1001 : audit 0 from='client.? 172.21.15.145:0/1069444538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:56.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[23614]: cluster 2024-03-07T03:12:56.238109+0000 mon.a (mon.0) 1002 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:12:56.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[23614]: cluster 2024-03-07T03:12:56.238519+0000 mon.a (mon.0) 1003 : cluster 0 Standby manager daemon x started 2024-03-07T03:12:56.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[23614]: audit 2024-03-07T03:12:56.238554+0000 mon.b (mon.1) 31 : audit 0 from='mgr.? 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:12:56.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[23614]: audit 2024-03-07T03:12:56.239552+0000 mon.b (mon.1) 32 : audit 0 from='mgr.? 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:12:56.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[23614]: audit 2024-03-07T03:12:56.242705+0000 mon.b (mon.1) 33 : audit 0 from='mgr.? 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:12:56.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:56 smithi189 bash[23614]: audit 2024-03-07T03:12:56.244447+0000 mon.b (mon.1) 34 : audit 0 from='mgr.? 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:12:57.490 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:12:57 smithi145 bash[57902]: ts=2024-03-07T03:12:57.190Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=5 2024-03-07T03:12:57.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:57 smithi145 bash[30906]: cluster 2024-03-07T03:12:56.503210+0000 mgr.y (mgr.24302) 259 : cluster 0 pgmap v314: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 121 KiB/s rd, 4.0 MiB/s wr, 173 op/s 2024-03-07T03:12:57.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:57 smithi145 bash[30906]: cluster 2024-03-07T03:12:56.505197+0000 mon.a (mon.0) 1004 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2024-03-07T03:12:57.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:57 smithi145 bash[30906]: cluster 2024-03-07T03:12:56.506503+0000 mon.a (mon.0) 1005 : cluster 0 mgrmap e21: y(active, since 6m), standbys: x 2024-03-07T03:12:57.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:57 smithi145 bash[21339]: cluster 2024-03-07T03:12:56.503210+0000 mgr.y (mgr.24302) 259 : cluster 0 pgmap v314: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 121 KiB/s rd, 4.0 MiB/s wr, 173 op/s 2024-03-07T03:12:57.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:57 smithi145 bash[21339]: cluster 2024-03-07T03:12:56.505197+0000 mon.a (mon.0) 1004 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2024-03-07T03:12:57.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:57 smithi145 bash[21339]: cluster 2024-03-07T03:12:56.506503+0000 mon.a (mon.0) 1005 : cluster 0 mgrmap e21: y(active, since 6m), standbys: x 2024-03-07T03:12:57.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:57 smithi189 bash[23614]: cluster 2024-03-07T03:12:56.503210+0000 mgr.y (mgr.24302) 259 : cluster 0 pgmap v314: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 121 KiB/s rd, 4.0 MiB/s wr, 173 op/s 2024-03-07T03:12:57.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:57 smithi189 bash[23614]: cluster 2024-03-07T03:12:56.505197+0000 mon.a (mon.0) 1004 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2024-03-07T03:12:57.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:57 smithi189 bash[23614]: cluster 2024-03-07T03:12:56.506503+0000 mon.a (mon.0) 1005 : cluster 0 mgrmap e21: y(active, since 6m), standbys: x 2024-03-07T03:12:58.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:58 smithi145 bash[30906]: cluster 2024-03-07T03:12:57.509609+0000 mon.a (mon.0) 1006 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2024-03-07T03:12:58.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:58 smithi145 bash[30906]: audit 2024-03-07T03:12:58.089692+0000 mon.a (mon.0) 1007 : audit 0 from='client.? 172.21.15.145:0/3229599035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:58.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:58 smithi145 bash[21339]: cluster 2024-03-07T03:12:57.509609+0000 mon.a (mon.0) 1006 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2024-03-07T03:12:58.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:58 smithi145 bash[21339]: audit 2024-03-07T03:12:58.089692+0000 mon.a (mon.0) 1007 : audit 0 from='client.? 172.21.15.145:0/3229599035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:58.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:58 smithi189 bash[23614]: cluster 2024-03-07T03:12:57.509609+0000 mon.a (mon.0) 1006 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2024-03-07T03:12:58.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:58 smithi189 bash[23614]: audit 2024-03-07T03:12:58.089692+0000 mon.a (mon.0) 1007 : audit 0 from='client.? 172.21.15.145:0/3229599035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:12:58.988 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.SnapsShrinkAfterPrepare (12935 ms) 2024-03-07T03:12:58.989 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroBeforePrepare 2024-03-07T03:12:59.070 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:12:59.102 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:12:59.213 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:12:59.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:59 smithi145 bash[21339]: cluster 2024-03-07T03:12:58.504152+0000 mgr.y (mgr.24302) 260 : cluster 0 pgmap v317: 137 pgs: 137 active+clean; 465 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 37 MiB/s rd, 5.0 MiB/s wr, 395 op/s 2024-03-07T03:12:59.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:12:59 smithi145 bash[21339]: cluster 2024-03-07T03:12:58.505928+0000 mon.a (mon.0) 1008 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2024-03-07T03:12:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:59 smithi145 bash[30906]: cluster 2024-03-07T03:12:58.504152+0000 mgr.y (mgr.24302) 260 : cluster 0 pgmap v317: 137 pgs: 137 active+clean; 465 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 37 MiB/s rd, 5.0 MiB/s wr, 395 op/s 2024-03-07T03:12:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:12:59 smithi145 bash[30906]: cluster 2024-03-07T03:12:58.505928+0000 mon.a (mon.0) 1008 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2024-03-07T03:12:59.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:59 smithi189 bash[23614]: cluster 2024-03-07T03:12:58.504152+0000 mgr.y (mgr.24302) 260 : cluster 0 pgmap v317: 137 pgs: 137 active+clean; 465 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 37 MiB/s rd, 5.0 MiB/s wr, 395 op/s 2024-03-07T03:12:59.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:12:59 smithi189 bash[23614]: cluster 2024-03-07T03:12:58.505928+0000 mon.a (mon.0) 1008 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2024-03-07T03:13:00.546 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 8388609~8388609 2024-03-07T03:13:00.564 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 0~1000 2024-03-07T03:13:00.564 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 1010~1000 2024-03-07T03:13:00.564 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:13:00.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:00 smithi145 bash[21339]: cluster 2024-03-07T03:12:59.519440+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2024-03-07T03:13:00.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:00 smithi145 bash[30906]: cluster 2024-03-07T03:12:59.519440+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2024-03-07T03:13:00.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:00 smithi189 bash[23614]: cluster 2024-03-07T03:12:59.519440+0000 mon.a (mon.0) 1009 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2024-03-07T03:13:01.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:01 smithi189 bash[23614]: cluster 2024-03-07T03:13:00.504911+0000 mgr.y (mgr.24302) 261 : cluster 0 pgmap v319: 137 pgs: 137 active+clean; 469 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 439 op/s 2024-03-07T03:13:01.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:01 smithi189 bash[23614]: audit 2024-03-07T03:13:00.527451+0000 mon.a (mon.0) 1010 : audit 0 from='client.? 172.21.15.145:0/431849560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:01.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:01 smithi189 bash[23614]: cluster 2024-03-07T03:13:00.541980+0000 mon.a (mon.0) 1011 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2024-03-07T03:13:01.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:01 smithi145 bash[21339]: cluster 2024-03-07T03:13:00.504911+0000 mgr.y (mgr.24302) 261 : cluster 0 pgmap v319: 137 pgs: 137 active+clean; 469 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 439 op/s 2024-03-07T03:13:01.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:01 smithi145 bash[21339]: audit 2024-03-07T03:13:00.527451+0000 mon.a (mon.0) 1010 : audit 0 from='client.? 172.21.15.145:0/431849560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:01.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:01 smithi145 bash[21339]: cluster 2024-03-07T03:13:00.541980+0000 mon.a (mon.0) 1011 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2024-03-07T03:13:01.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:01 smithi145 bash[30906]: cluster 2024-03-07T03:13:00.504911+0000 mgr.y (mgr.24302) 261 : cluster 0 pgmap v319: 137 pgs: 137 active+clean; 469 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 439 op/s 2024-03-07T03:13:01.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:01 smithi145 bash[30906]: audit 2024-03-07T03:13:00.527451+0000 mon.a (mon.0) 1010 : audit 0 from='client.? 172.21.15.145:0/431849560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:01.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:01 smithi145 bash[30906]: cluster 2024-03-07T03:13:00.541980+0000 mon.a (mon.0) 1011 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2024-03-07T03:13:02.580 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 8388609~1000 2024-03-07T03:13:02.589 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 16777228~1000 2024-03-07T03:13:02.589 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 167772160 2024-03-07T03:13:02.603 INFO:tasks.workunit.client.0.smithi145.stdout:write: 2 83886079~8388609 2024-03-07T03:13:02.644 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:13:02.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:02 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:13:02] "GET /metrics HTTP/1.1" 200 38623 "" "Prometheus/2.43.0" 2024-03-07T03:13:02.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:02 smithi145 bash[21339]: cluster 2024-03-07T03:13:01.559161+0000 mon.a (mon.0) 1012 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2024-03-07T03:13:02.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:02 smithi145 bash[21339]: audit 2024-03-07T03:13:02.456021+0000 mon.c (mon.2) 151 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:02.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:02 smithi145 bash[30906]: cluster 2024-03-07T03:13:01.559161+0000 mon.a (mon.0) 1012 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2024-03-07T03:13:02.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:02 smithi145 bash[30906]: audit 2024-03-07T03:13:02.456021+0000 mon.c (mon.2) 151 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:02.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:02 smithi189 bash[23614]: cluster 2024-03-07T03:13:01.559161+0000 mon.a (mon.0) 1012 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2024-03-07T03:13:02.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:02 smithi189 bash[23614]: audit 2024-03-07T03:13:02.456021+0000 mon.c (mon.2) 151 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:03.455 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:13:03.484 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 83886079~1 2024-03-07T03:13:03.493 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 0 2024-03-07T03:13:03.513 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:13:03.647 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: [07/Mar/2024:03:13:03] ENGINE Bus STOPPING 2024-03-07T03:13:03.647 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: [07/Mar/2024:03:13:03] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:13:03.647 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: [07/Mar/2024:03:13:03] ENGINE Bus STOPPED 2024-03-07T03:13:03.647 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: [07/Mar/2024:03:13:03] ENGINE Bus STARTING 2024-03-07T03:13:03.647 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: debug 2024-03-07T03:13:03.611+0000 7f96c1ea2700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:03.647 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: debug 2024-03-07T03:13:03.611+0000 7f96c1ea2700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:03.648 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: debug 2024-03-07T03:13:03.611+0000 7f96c1ea2700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:03.648 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: debug 2024-03-07T03:13:03.611+0000 7f96c1ea2700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:03.648 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: debug 2024-03-07T03:13:03.611+0000 7f96c1ea2700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:03.648 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: debug 2024-03-07T03:13:03.611+0000 7f96c3ea6700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:03.648 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: debug 2024-03-07T03:13:03.611+0000 7f96c3ea6700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:03.648 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: debug 2024-03-07T03:13:03.611+0000 7f96c3ea6700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:03.648 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: debug 2024-03-07T03:13:03.611+0000 7f96c3ea6700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:03.648 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: debug 2024-03-07T03:13:03.611+0000 7f96c3ea6700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:03.648 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: [07/Mar/2024:03:13:03] ENGINE Serving on http://:::9283 2024-03-07T03:13:03.648 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: cluster 2024-03-07T03:13:02.569356+0000 mon.a (mon.0) 1013 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:02.994758+0000 mon.c (mon.2) 152 : audit 0 from='client.? 172.21.15.145:0/2982968303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.235954+0000 mon.a (mon.0) 1014 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.254152+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.258325+0000 mon.c (mon.2) 153 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.260348+0000 mon.c (mon.2) 154 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.272150+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.356458+0000 mon.c (mon.2) 155 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.365731+0000 mon.c (mon.2) 156 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.366496+0000 mon.a (mon.0) 1017 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.383019+0000 mon.b (mon.1) 35 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.383525+0000 mon.b (mon.1) 36 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.384140+0000 mon.b (mon.1) 37 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.384690+0000 mon.b (mon.1) 38 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:13:03.649 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: cluster 2024-03-07T03:13:03.384705+0000 mon.a (mon.0) 1018 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2024-03-07T03:13:03.650 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.385314+0000 mon.b (mon.1) 39 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.zkinbe"}]: dispatch 2024-03-07T03:13:03.650 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.386253+0000 mon.b (mon.1) 40 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:13:03.650 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.386864+0000 mon.a (mon.0) 1019 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-07T03:13:03.650 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: cluster 2024-03-07T03:13:03.386947+0000 mon.a (mon.0) 1020 : cluster 0 mgrmap e22: x(active, starting, since 0.0195591s) 2024-03-07T03:13:03.650 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.387059+0000 mon.b (mon.1) 41 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:13:03.650 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.387475+0000 mon.b (mon.1) 42 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:13:03.650 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.387832+0000 mon.b (mon.1) 43 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:13:03.650 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.388189+0000 mon.b (mon.1) 44 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:13:03.650 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.388645+0000 mon.b (mon.1) 45 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:13:03.650 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.389051+0000 mon.b (mon.1) 46 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:13:03.650 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.389552+0000 mon.b (mon.1) 47 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:13:03.650 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.390135+0000 mon.b (mon.1) 48 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:13:03.650 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.392749+0000 mon.b (mon.1) 49 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:13:03.651 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.393258+0000 mon.b (mon.1) 50 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:13:03.651 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.393969+0000 mon.b (mon.1) 51 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:13:03.651 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: cluster 2024-03-07T03:13:03.441819+0000 mon.a (mon.0) 1021 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2024-03-07T03:13:03.651 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: cluster 2024-03-07T03:13:03.445139+0000 mon.a (mon.0) 1022 : cluster 1 Manager daemon x is now available 2024-03-07T03:13:03.651 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.489183+0000 mon.b (mon.1) 52 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:03.651 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.493184+0000 mon.b (mon.1) 53 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:03.651 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.535617+0000 mon.b (mon.1) 54 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:13:03.651 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.537393+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:13:03.651 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.550559+0000 mon.b (mon.1) 55 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:03.651 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.555464+0000 mon.b (mon.1) 56 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-07T03:13:03.651 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[23614]: audit 2024-03-07T03:13:03.557261+0000 mon.a (mon.0) 1024 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-07T03:13:03.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21623]: ignoring --setuser ceph since I am not root 2024-03-07T03:13:03.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21623]: ignoring --setgroup ceph since I am not root 2024-03-07T03:13:03.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21623]: debug 2024-03-07T03:13:03.626+0000 7fdb1b2b8200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:13:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: cluster 2024-03-07T03:13:02.569356+0000 mon.a (mon.0) 1013 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2024-03-07T03:13:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:02.994758+0000 mon.c (mon.2) 152 : audit 0 from='client.? 172.21.15.145:0/2982968303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.235954+0000 mon.a (mon.0) 1014 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:13:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.254152+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:13:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.258325+0000 mon.c (mon.2) 153 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.260348+0000 mon.c (mon.2) 154 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.272150+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.356458+0000 mon.c (mon.2) 155 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.365731+0000 mon.c (mon.2) 156 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.366496+0000 mon.a (mon.0) 1017 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.383019+0000 mon.b (mon.1) 35 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.383525+0000 mon.b (mon.1) 36 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.384140+0000 mon.b (mon.1) 37 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.384690+0000 mon.b (mon.1) 38 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: cluster 2024-03-07T03:13:03.384705+0000 mon.a (mon.0) 1018 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.385314+0000 mon.b (mon.1) 39 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.zkinbe"}]: dispatch 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.386253+0000 mon.b (mon.1) 40 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.386864+0000 mon.a (mon.0) 1019 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: cluster 2024-03-07T03:13:03.386947+0000 mon.a (mon.0) 1020 : cluster 0 mgrmap e22: x(active, starting, since 0.0195591s) 2024-03-07T03:13:03.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.387059+0000 mon.b (mon.1) 41 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:13:03.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: cluster 2024-03-07T03:13:02.569356+0000 mon.a (mon.0) 1013 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2024-03-07T03:13:03.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:02.994758+0000 mon.c (mon.2) 152 : audit 0 from='client.? 172.21.15.145:0/2982968303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:03.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.235954+0000 mon.a (mon.0) 1014 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:13:03.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.254152+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:13:03.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.258325+0000 mon.c (mon.2) 153 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:03.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.260348+0000 mon.c (mon.2) 154 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:03.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.272150+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.24302 ' entity='mgr.y' 2024-03-07T03:13:03.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.356458+0000 mon.c (mon.2) 155 : audit 0 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:03.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.365731+0000 mon.c (mon.2) 156 : audit 1 from='mgr.24302 172.21.15.145:0/1465723018' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.366496+0000 mon.a (mon.0) 1017 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.383019+0000 mon.b (mon.1) 35 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.383525+0000 mon.b (mon.1) 36 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.384140+0000 mon.b (mon.1) 37 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.384690+0000 mon.b (mon.1) 38 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: cluster 2024-03-07T03:13:03.384705+0000 mon.a (mon.0) 1018 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.385314+0000 mon.b (mon.1) 39 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.zkinbe"}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.386253+0000 mon.b (mon.1) 40 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.386864+0000 mon.a (mon.0) 1019 : audit 1 from='mgr.24302 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: cluster 2024-03-07T03:13:03.386947+0000 mon.a (mon.0) 1020 : cluster 0 mgrmap e22: x(active, starting, since 0.0195591s) 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.387059+0000 mon.b (mon.1) 41 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.387475+0000 mon.b (mon.1) 42 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.387832+0000 mon.b (mon.1) 43 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.388189+0000 mon.b (mon.1) 44 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.388645+0000 mon.b (mon.1) 45 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.389051+0000 mon.b (mon.1) 46 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:13:03.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.389552+0000 mon.b (mon.1) 47 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:13:03.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.390135+0000 mon.b (mon.1) 48 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:13:03.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.392749+0000 mon.b (mon.1) 49 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:13:03.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.393258+0000 mon.b (mon.1) 50 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:13:03.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.393969+0000 mon.b (mon.1) 51 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:13:03.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: cluster 2024-03-07T03:13:03.441819+0000 mon.a (mon.0) 1021 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2024-03-07T03:13:03.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: cluster 2024-03-07T03:13:03.445139+0000 mon.a (mon.0) 1022 : cluster 1 Manager daemon x is now available 2024-03-07T03:13:03.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.489183+0000 mon.b (mon.1) 52 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:03.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.493184+0000 mon.b (mon.1) 53 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:03.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.535617+0000 mon.b (mon.1) 54 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:13:03.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.537393+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:13:03.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.550559+0000 mon.b (mon.1) 55 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:03.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.555464+0000 mon.b (mon.1) 56 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-07T03:13:03.814 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[30906]: audit 2024-03-07T03:13:03.557261+0000 mon.a (mon.0) 1024 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.387475+0000 mon.b (mon.1) 42 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.387832+0000 mon.b (mon.1) 43 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.388189+0000 mon.b (mon.1) 44 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.388645+0000 mon.b (mon.1) 45 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.389051+0000 mon.b (mon.1) 46 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.389552+0000 mon.b (mon.1) 47 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.390135+0000 mon.b (mon.1) 48 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.392749+0000 mon.b (mon.1) 49 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.393258+0000 mon.b (mon.1) 50 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.393969+0000 mon.b (mon.1) 51 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: cluster 2024-03-07T03:13:03.441819+0000 mon.a (mon.0) 1021 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: cluster 2024-03-07T03:13:03.445139+0000 mon.a (mon.0) 1022 : cluster 1 Manager daemon x is now available 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.489183+0000 mon.b (mon.1) 52 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:03.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.493184+0000 mon.b (mon.1) 53 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:03.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.535617+0000 mon.b (mon.1) 54 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:13:03.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.537393+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:13:03.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.550559+0000 mon.b (mon.1) 55 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:03.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.555464+0000 mon.b (mon.1) 56 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-07T03:13:03.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:03 smithi145 bash[21339]: audit 2024-03-07T03:13:03.557261+0000 mon.a (mon.0) 1024 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-07T03:13:04.081 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:03 smithi189 bash[52103]: [07/Mar/2024:03:13:03] ENGINE Bus STARTED 2024-03-07T03:13:05.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[21339]: cephadm 2024-03-07T03:13:04.002027+0000 mgr.x (mgr.24511) 1 : cephadm 1 Deploying cephadm binary to smithi189 2024-03-07T03:13:05.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[21339]: cephadm 2024-03-07T03:13:04.013442+0000 mgr.x (mgr.24511) 2 : cephadm 1 Deploying cephadm binary to smithi145 2024-03-07T03:13:05.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[21339]: cephadm 2024-03-07T03:13:04.402064+0000 mgr.x (mgr.24511) 3 : cephadm 1 [07/Mar/2024:03:13:04] ENGINE Bus STARTING 2024-03-07T03:13:05.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[21339]: cluster 2024-03-07T03:13:04.449978+0000 mon.a (mon.0) 1025 : cluster 0 mgrmap e23: x(active, since 1.08258s) 2024-03-07T03:13:05.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[21339]: cluster 2024-03-07T03:13:04.475633+0000 mgr.x (mgr.24511) 4 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:13:05.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[21339]: cephadm 2024-03-07T03:13:04.540533+0000 mgr.x (mgr.24511) 5 : cephadm 1 [07/Mar/2024:03:13:04] ENGINE Serving on http://172.21.15.189:8765 2024-03-07T03:13:05.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[21339]: cluster 2024-03-07T03:13:04.570023+0000 mon.a (mon.0) 1026 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2024-03-07T03:13:05.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[21339]: cephadm 2024-03-07T03:13:04.711216+0000 mgr.x (mgr.24511) 6 : cephadm 1 [07/Mar/2024:03:13:04] ENGINE Serving on https://172.21.15.189:7150 2024-03-07T03:13:05.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[21339]: cephadm 2024-03-07T03:13:04.711489+0000 mgr.x (mgr.24511) 7 : cephadm 1 [07/Mar/2024:03:13:04] ENGINE Bus STARTED 2024-03-07T03:13:05.560 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[21623]: debug 2024-03-07T03:13:05.265+0000 7fdb1b2b8200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:13:05.560 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[21623]: debug 2024-03-07T03:13:05.369+0000 7fdb1b2b8200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:13:05.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[30906]: cephadm 2024-03-07T03:13:04.002027+0000 mgr.x (mgr.24511) 1 : cephadm 1 Deploying cephadm binary to smithi189 2024-03-07T03:13:05.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[30906]: cephadm 2024-03-07T03:13:04.013442+0000 mgr.x (mgr.24511) 2 : cephadm 1 Deploying cephadm binary to smithi145 2024-03-07T03:13:05.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[30906]: cephadm 2024-03-07T03:13:04.402064+0000 mgr.x (mgr.24511) 3 : cephadm 1 [07/Mar/2024:03:13:04] ENGINE Bus STARTING 2024-03-07T03:13:05.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[30906]: cluster 2024-03-07T03:13:04.449978+0000 mon.a (mon.0) 1025 : cluster 0 mgrmap e23: x(active, since 1.08258s) 2024-03-07T03:13:05.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[30906]: cluster 2024-03-07T03:13:04.475633+0000 mgr.x (mgr.24511) 4 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:13:05.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[30906]: cephadm 2024-03-07T03:13:04.540533+0000 mgr.x (mgr.24511) 5 : cephadm 1 [07/Mar/2024:03:13:04] ENGINE Serving on http://172.21.15.189:8765 2024-03-07T03:13:05.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[30906]: cluster 2024-03-07T03:13:04.570023+0000 mon.a (mon.0) 1026 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2024-03-07T03:13:05.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[30906]: cephadm 2024-03-07T03:13:04.711216+0000 mgr.x (mgr.24511) 6 : cephadm 1 [07/Mar/2024:03:13:04] ENGINE Serving on https://172.21.15.189:7150 2024-03-07T03:13:05.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[30906]: cephadm 2024-03-07T03:13:04.711489+0000 mgr.x (mgr.24511) 7 : cephadm 1 [07/Mar/2024:03:13:04] ENGINE Bus STARTED 2024-03-07T03:13:05.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:05 smithi189 bash[23614]: cephadm 2024-03-07T03:13:04.002027+0000 mgr.x (mgr.24511) 1 : cephadm 1 Deploying cephadm binary to smithi189 2024-03-07T03:13:05.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:05 smithi189 bash[23614]: cephadm 2024-03-07T03:13:04.013442+0000 mgr.x (mgr.24511) 2 : cephadm 1 Deploying cephadm binary to smithi145 2024-03-07T03:13:05.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:05 smithi189 bash[23614]: cephadm 2024-03-07T03:13:04.402064+0000 mgr.x (mgr.24511) 3 : cephadm 1 [07/Mar/2024:03:13:04] ENGINE Bus STARTING 2024-03-07T03:13:05.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:05 smithi189 bash[23614]: cluster 2024-03-07T03:13:04.449978+0000 mon.a (mon.0) 1025 : cluster 0 mgrmap e23: x(active, since 1.08258s) 2024-03-07T03:13:05.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:05 smithi189 bash[23614]: cluster 2024-03-07T03:13:04.475633+0000 mgr.x (mgr.24511) 4 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:13:05.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:05 smithi189 bash[23614]: cephadm 2024-03-07T03:13:04.540533+0000 mgr.x (mgr.24511) 5 : cephadm 1 [07/Mar/2024:03:13:04] ENGINE Serving on http://172.21.15.189:8765 2024-03-07T03:13:05.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:05 smithi189 bash[23614]: cluster 2024-03-07T03:13:04.570023+0000 mon.a (mon.0) 1026 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2024-03-07T03:13:05.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:05 smithi189 bash[23614]: cephadm 2024-03-07T03:13:04.711216+0000 mgr.x (mgr.24511) 6 : cephadm 1 [07/Mar/2024:03:13:04] ENGINE Serving on https://172.21.15.189:7150 2024-03-07T03:13:05.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:05 smithi189 bash[23614]: cephadm 2024-03-07T03:13:04.711489+0000 mgr.x (mgr.24511) 7 : cephadm 1 [07/Mar/2024:03:13:04] ENGINE Bus STARTED 2024-03-07T03:13:06.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:05 smithi145 bash[21623]: debug 2024-03-07T03:13:05.637+0000 7fdb1b2b8200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:13:06.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:06 smithi145 bash[30906]: cluster 2024-03-07T03:13:05.392827+0000 mgr.x (mgr.24511) 8 : cluster 0 pgmap v6: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:13:06.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:06 smithi145 bash[30906]: audit 2024-03-07T03:13:05.519763+0000 mon.c (mon.2) 157 : audit 0 from='client.? 172.21.15.145:0/3705914632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:06.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:06 smithi145 bash[30906]: cluster 2024-03-07T03:13:05.571667+0000 mon.a (mon.0) 1027 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2024-03-07T03:13:06.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:06 smithi145 bash[21339]: cluster 2024-03-07T03:13:05.392827+0000 mgr.x (mgr.24511) 8 : cluster 0 pgmap v6: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:13:06.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:06 smithi145 bash[21339]: audit 2024-03-07T03:13:05.519763+0000 mon.c (mon.2) 157 : audit 0 from='client.? 172.21.15.145:0/3705914632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:06.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:06 smithi145 bash[21339]: cluster 2024-03-07T03:13:05.571667+0000 mon.a (mon.0) 1027 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2024-03-07T03:13:06.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:06 smithi145 bash[21623]: debug 2024-03-07T03:13:06.425+0000 7fdb1b2b8200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:13:06.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:06 smithi145 bash[21623]: debug 2024-03-07T03:13:06.541+0000 7fdb1b2b8200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:13:06.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:06 smithi145 bash[21623]: debug 2024-03-07T03:13:06.657+0000 7fdb1b2b8200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:13:06.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:06 smithi189 bash[23614]: cluster 2024-03-07T03:13:05.392827+0000 mgr.x (mgr.24511) 8 : cluster 0 pgmap v6: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:13:06.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:06 smithi189 bash[23614]: audit 2024-03-07T03:13:05.519763+0000 mon.c (mon.2) 157 : audit 0 from='client.? 172.21.15.145:0/3705914632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:06.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:06 smithi189 bash[23614]: cluster 2024-03-07T03:13:05.571667+0000 mon.a (mon.0) 1027 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2024-03-07T03:13:07.172 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:13:07.334 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:07 smithi145 bash[21623]: debug 2024-03-07T03:13:07.081+0000 7fdb1b2b8200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:13:07.335 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:07 smithi145 bash[21623]: debug 2024-03-07T03:13:07.209+0000 7fdb1b2b8200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:13:07.435 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:13:07.732 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:07 smithi145 bash[21339]: cluster 2024-03-07T03:13:06.575336+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2024-03-07T03:13:07.733 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:07 smithi145 bash[21339]: cluster 2024-03-07T03:13:06.589941+0000 mon.a (mon.0) 1029 : cluster 0 mgrmap e24: x(active, since 3s) 2024-03-07T03:13:07.733 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:07 smithi145 bash[21623]: debug 2024-03-07T03:13:07.345+0000 7fdb1b2b8200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:13:07.733 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:07 smithi145 bash[30906]: cluster 2024-03-07T03:13:06.575336+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2024-03-07T03:13:07.733 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:07 smithi145 bash[30906]: cluster 2024-03-07T03:13:06.589941+0000 mon.a (mon.0) 1029 : cluster 0 mgrmap e24: x(active, since 3s) 2024-03-07T03:13:07.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:07 smithi189 bash[23614]: cluster 2024-03-07T03:13:06.575336+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2024-03-07T03:13:07.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:07 smithi189 bash[23614]: cluster 2024-03-07T03:13:06.589941+0000 mon.a (mon.0) 1029 : cluster 0 mgrmap e24: x(active, since 3s) 2024-03-07T03:13:07.984 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:07 smithi145 bash[21623]: debug 2024-03-07T03:13:07.741+0000 7fdb1b2b8200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:13:08.246 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:07 smithi145 bash[21623]: debug 2024-03-07T03:13:07.993+0000 7fdb1b2b8200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:13:08.559 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:08 smithi145 bash[21623]: debug 2024-03-07T03:13:08.257+0000 7fdb1b2b8200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:13:08.559 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:08 smithi145 bash[21623]: debug 2024-03-07T03:13:08.369+0000 7fdb1b2b8200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:13:08.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:08 smithi189 bash[23614]: cluster 2024-03-07T03:13:07.393752+0000 mgr.x (mgr.24511) 9 : cluster 0 pgmap v9: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:13:08.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:08 smithi189 bash[23614]: cluster 2024-03-07T03:13:07.586397+0000 mon.a (mon.0) 1030 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2024-03-07T03:13:08.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:08 smithi189 bash[23614]: audit 2024-03-07T03:13:07.957996+0000 mon.a (mon.0) 1031 : audit 0 from='client.? 172.21.15.145:0/2308986346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:08.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:08 smithi189 bash[23614]: cluster 2024-03-07T03:13:08.448001+0000 mon.a (mon.0) 1032 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2024-03-07T03:13:08.895 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:08 smithi145 bash[21623]: debug 2024-03-07T03:13:08.657+0000 7fdb1b2b8200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:13:08.895 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:08 smithi145 bash[21339]: cluster 2024-03-07T03:13:07.393752+0000 mgr.x (mgr.24511) 9 : cluster 0 pgmap v9: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:13:08.895 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:08 smithi145 bash[21339]: cluster 2024-03-07T03:13:07.586397+0000 mon.a (mon.0) 1030 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2024-03-07T03:13:08.895 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:08 smithi145 bash[21339]: audit 2024-03-07T03:13:07.957996+0000 mon.a (mon.0) 1031 : audit 0 from='client.? 172.21.15.145:0/2308986346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:08.895 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:08 smithi145 bash[21339]: cluster 2024-03-07T03:13:08.448001+0000 mon.a (mon.0) 1032 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2024-03-07T03:13:08.895 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:08 smithi145 bash[30906]: cluster 2024-03-07T03:13:07.393752+0000 mgr.x (mgr.24511) 9 : cluster 0 pgmap v9: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:13:08.895 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:08 smithi145 bash[30906]: cluster 2024-03-07T03:13:07.586397+0000 mon.a (mon.0) 1030 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2024-03-07T03:13:08.895 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:08 smithi145 bash[30906]: audit 2024-03-07T03:13:07.957996+0000 mon.a (mon.0) 1031 : audit 0 from='client.? 172.21.15.145:0/2308986346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:08.895 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:08 smithi145 bash[30906]: cluster 2024-03-07T03:13:08.448001+0000 mon.a (mon.0) 1032 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2024-03-07T03:13:09.306 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:08 smithi145 bash[21623]: debug 2024-03-07T03:13:08.905+0000 7fdb1b2b8200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:13:09.559 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:09 smithi145 bash[21623]: debug 2024-03-07T03:13:09.317+0000 7fdb1b2b8200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:13:09.559 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:09 smithi145 bash[21623]: debug 2024-03-07T03:13:09.437+0000 7fdb1b2b8200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:13:09.789 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.SnapsShrinkToZeroBeforePrepare (10801 ms) 2024-03-07T03:13:09.789 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroAfterPrepare 2024-03-07T03:13:09.857 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:13:09.878 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:13:09.983 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:13:10.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:09 smithi145 bash[21623]: debug 2024-03-07T03:13:09.745+0000 7fdb1b2b8200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:13:10.799 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:10 smithi145 bash[21339]: cluster 2024-03-07T03:13:09.394514+0000 mgr.x (mgr.24511) 10 : cluster 0 pgmap v12: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:13:10.799 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:10 smithi145 bash[21339]: cluster 2024-03-07T03:13:09.469344+0000 mon.a (mon.0) 1033 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2024-03-07T03:13:10.799 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:10 smithi145 bash[21339]: audit 2024-03-07T03:13:10.390450+0000 mon.c (mon.2) 158 : audit 0 from='client.? 172.21.15.145:0/2481790822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:10.799 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:10 smithi145 bash[30906]: cluster 2024-03-07T03:13:09.394514+0000 mgr.x (mgr.24511) 10 : cluster 0 pgmap v12: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:13:10.800 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:10 smithi145 bash[30906]: cluster 2024-03-07T03:13:09.469344+0000 mon.a (mon.0) 1033 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2024-03-07T03:13:10.800 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:10 smithi145 bash[30906]: audit 2024-03-07T03:13:10.390450+0000 mon.c (mon.2) 158 : audit 0 from='client.? 172.21.15.145:0/2481790822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:10.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:10 smithi189 bash[23614]: cluster 2024-03-07T03:13:09.394514+0000 mgr.x (mgr.24511) 10 : cluster 0 pgmap v12: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:13:10.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:10 smithi189 bash[23614]: cluster 2024-03-07T03:13:09.469344+0000 mon.a (mon.0) 1033 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2024-03-07T03:13:10.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:10 smithi189 bash[23614]: audit 2024-03-07T03:13:10.390450+0000 mon.c (mon.2) 158 : audit 0 from='client.? 172.21.15.145:0/2481790822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:11.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:10 smithi145 bash[21623]: debug 2024-03-07T03:13:10.805+0000 7fdb1b2b8200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:13:11.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:10 smithi145 bash[21623]: debug 2024-03-07T03:13:10.925+0000 7fdb1b2b8200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:13:11.559 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:11 smithi145 bash[21623]: debug 2024-03-07T03:13:11.221+0000 7fdb1b2b8200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:13:11.559 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:11 smithi145 bash[21623]: debug 2024-03-07T03:13:11.341+0000 7fdb1b2b8200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:13:11.609 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 8388609~8388609 2024-03-07T03:13:11.644 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 0~1000 2024-03-07T03:13:11.644 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 1010~1000 2024-03-07T03:13:11.644 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:13:11.947 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:11 smithi145 bash[30906]: cluster 2024-03-07T03:13:10.598338+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2024-03-07T03:13:11.947 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:11 smithi145 bash[21339]: cluster 2024-03-07T03:13:10.598338+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2024-03-07T03:13:11.947 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:11 smithi145 bash[21623]: debug 2024-03-07T03:13:11.833+0000 7fdb1b2b8200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:13:12.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:11 smithi189 bash[23614]: cluster 2024-03-07T03:13:10.598338+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2024-03-07T03:13:12.309 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:11 smithi145 bash[21623]: debug 2024-03-07T03:13:11.957+0000 7fdb1b2b8200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:13:12.310 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:11 smithi145 bash[21623]: [07/Mar/2024:03:13:11] ENGINE Bus STARTING 2024-03-07T03:13:12.310 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[21623]: CherryPy Checker: 2024-03-07T03:13:12.310 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[21623]: The Application mounted at '' has an empty config. 2024-03-07T03:13:12.310 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[21623]: [07/Mar/2024:03:13:12] ENGINE Serving on http://:::9283 2024-03-07T03:13:12.310 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[21623]: [07/Mar/2024:03:13:12] ENGINE Bus STARTED 2024-03-07T03:13:12.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:13:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-07T03:13:12.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[21339]: cluster 2024-03-07T03:13:11.395261+0000 mgr.x (mgr.24511) 11 : cluster 0 pgmap v15: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 513 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 431 op/s 2024-03-07T03:13:12.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[21339]: cluster 2024-03-07T03:13:11.602234+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2024-03-07T03:13:12.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[21339]: cluster 2024-03-07T03:13:11.981545+0000 mon.a (mon.0) 1036 : cluster 0 Standby manager daemon y started 2024-03-07T03:13:12.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[21339]: audit 2024-03-07T03:13:11.981766+0000 mon.c (mon.2) 159 : audit 0 from='mgr.? 172.21.15.145:0/3084736328' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-07T03:13:12.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[21339]: audit 2024-03-07T03:13:11.986492+0000 mon.c (mon.2) 160 : audit 0 from='mgr.? 172.21.15.145:0/3084736328' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:13:12.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[21339]: audit 2024-03-07T03:13:11.988359+0000 mon.c (mon.2) 161 : audit 0 from='mgr.? 172.21.15.145:0/3084736328' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-07T03:13:12.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[21339]: audit 2024-03-07T03:13:11.988841+0000 mon.c (mon.2) 162 : audit 0 from='mgr.? 172.21.15.145:0/3084736328' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:13:12.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[30906]: cluster 2024-03-07T03:13:11.395261+0000 mgr.x (mgr.24511) 11 : cluster 0 pgmap v15: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 513 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 431 op/s 2024-03-07T03:13:12.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[30906]: cluster 2024-03-07T03:13:11.602234+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2024-03-07T03:13:12.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[30906]: cluster 2024-03-07T03:13:11.981545+0000 mon.a (mon.0) 1036 : cluster 0 Standby manager daemon y started 2024-03-07T03:13:12.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[30906]: audit 2024-03-07T03:13:11.981766+0000 mon.c (mon.2) 159 : audit 0 from='mgr.? 172.21.15.145:0/3084736328' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-07T03:13:12.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[30906]: audit 2024-03-07T03:13:11.986492+0000 mon.c (mon.2) 160 : audit 0 from='mgr.? 172.21.15.145:0/3084736328' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:13:12.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[30906]: audit 2024-03-07T03:13:11.988359+0000 mon.c (mon.2) 161 : audit 0 from='mgr.? 172.21.15.145:0/3084736328' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-07T03:13:12.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:12 smithi145 bash[30906]: audit 2024-03-07T03:13:11.988841+0000 mon.c (mon.2) 162 : audit 0 from='mgr.? 172.21.15.145:0/3084736328' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:13:13.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:12 smithi189 bash[23614]: cluster 2024-03-07T03:13:11.395261+0000 mgr.x (mgr.24511) 11 : cluster 0 pgmap v15: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 513 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 431 op/s 2024-03-07T03:13:13.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:12 smithi189 bash[23614]: cluster 2024-03-07T03:13:11.602234+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2024-03-07T03:13:13.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:12 smithi189 bash[23614]: cluster 2024-03-07T03:13:11.981545+0000 mon.a (mon.0) 1036 : cluster 0 Standby manager daemon y started 2024-03-07T03:13:13.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:12 smithi189 bash[23614]: audit 2024-03-07T03:13:11.981766+0000 mon.c (mon.2) 159 : audit 0 from='mgr.? 172.21.15.145:0/3084736328' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-07T03:13:13.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:12 smithi189 bash[23614]: audit 2024-03-07T03:13:11.986492+0000 mon.c (mon.2) 160 : audit 0 from='mgr.? 172.21.15.145:0/3084736328' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:13:13.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:12 smithi189 bash[23614]: audit 2024-03-07T03:13:11.988359+0000 mon.c (mon.2) 161 : audit 0 from='mgr.? 172.21.15.145:0/3084736328' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-07T03:13:13.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:12 smithi189 bash[23614]: audit 2024-03-07T03:13:11.988841+0000 mon.c (mon.2) 162 : audit 0 from='mgr.? 172.21.15.145:0/3084736328' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:13:13.468 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 8388609~1000 2024-03-07T03:13:13.487 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 16777228~1000 2024-03-07T03:13:13.487 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 167772160 2024-03-07T03:13:13.507 INFO:tasks.workunit.client.0.smithi145.stdout:write: 2 83886079~8388609 2024-03-07T03:13:13.549 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:13:14.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:13 smithi145 bash[21339]: cluster 2024-03-07T03:13:12.630030+0000 mon.a (mon.0) 1037 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2024-03-07T03:13:14.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:13 smithi145 bash[21339]: cluster 2024-03-07T03:13:12.630946+0000 mon.a (mon.0) 1038 : cluster 0 mgrmap e25: x(active, since 9s), standbys: y 2024-03-07T03:13:14.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:13 smithi145 bash[21339]: audit 2024-03-07T03:13:12.631184+0000 mon.b (mon.1) 57 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:13:14.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:13 smithi145 bash[21339]: audit 2024-03-07T03:13:12.823727+0000 mon.c (mon.2) 163 : audit 0 from='client.? 172.21.15.145:0/2195625149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:14.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:13 smithi145 bash[21339]: cluster 2024-03-07T03:13:13.448743+0000 mon.a (mon.0) 1039 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2024-03-07T03:13:14.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:13 smithi145 bash[30906]: cluster 2024-03-07T03:13:12.630030+0000 mon.a (mon.0) 1037 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2024-03-07T03:13:14.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:13 smithi145 bash[30906]: cluster 2024-03-07T03:13:12.630946+0000 mon.a (mon.0) 1038 : cluster 0 mgrmap e25: x(active, since 9s), standbys: y 2024-03-07T03:13:14.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:13 smithi145 bash[30906]: audit 2024-03-07T03:13:12.631184+0000 mon.b (mon.1) 57 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:13:14.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:13 smithi145 bash[30906]: audit 2024-03-07T03:13:12.823727+0000 mon.c (mon.2) 163 : audit 0 from='client.? 172.21.15.145:0/2195625149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:14.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:13 smithi145 bash[30906]: cluster 2024-03-07T03:13:13.448743+0000 mon.a (mon.0) 1039 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2024-03-07T03:13:14.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:13 smithi189 bash[23614]: cluster 2024-03-07T03:13:12.630030+0000 mon.a (mon.0) 1037 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2024-03-07T03:13:14.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:13 smithi189 bash[23614]: cluster 2024-03-07T03:13:12.630946+0000 mon.a (mon.0) 1038 : cluster 0 mgrmap e25: x(active, since 9s), standbys: y 2024-03-07T03:13:14.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:13 smithi189 bash[23614]: audit 2024-03-07T03:13:12.631184+0000 mon.b (mon.1) 57 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:13:14.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:13 smithi189 bash[23614]: audit 2024-03-07T03:13:12.823727+0000 mon.c (mon.2) 163 : audit 0 from='client.? 172.21.15.145:0/2195625149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:14.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:13 smithi189 bash[23614]: cluster 2024-03-07T03:13:13.448743+0000 mon.a (mon.0) 1039 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2024-03-07T03:13:15.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:14 smithi145 bash[21339]: cluster 2024-03-07T03:13:13.396114+0000 mgr.x (mgr.24511) 12 : cluster 0 pgmap v18: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 513 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 431 op/s 2024-03-07T03:13:15.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:14 smithi145 bash[21339]: cluster 2024-03-07T03:13:14.622467+0000 mon.a (mon.0) 1040 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2024-03-07T03:13:15.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:14 smithi145 bash[30906]: cluster 2024-03-07T03:13:13.396114+0000 mgr.x (mgr.24511) 12 : cluster 0 pgmap v18: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 513 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 431 op/s 2024-03-07T03:13:15.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:14 smithi145 bash[30906]: cluster 2024-03-07T03:13:14.622467+0000 mon.a (mon.0) 1040 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2024-03-07T03:13:15.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:14 smithi189 bash[23614]: cluster 2024-03-07T03:13:13.396114+0000 mgr.x (mgr.24511) 12 : cluster 0 pgmap v18: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 513 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 431 op/s 2024-03-07T03:13:15.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:14 smithi189 bash[23614]: cluster 2024-03-07T03:13:14.622467+0000 mon.a (mon.0) 1040 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2024-03-07T03:13:15.652 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:13:15.707 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 83886079~1 2024-03-07T03:13:15.728 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:13:16.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:15 smithi145 bash[21339]: audit 2024-03-07T03:13:15.259917+0000 mon.a (mon.0) 1041 : audit 0 from='client.? 172.21.15.145:0/265997056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:16.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:15 smithi145 bash[30906]: audit 2024-03-07T03:13:15.259917+0000 mon.a (mon.0) 1041 : audit 0 from='client.? 172.21.15.145:0/265997056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:16.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:15 smithi189 bash[23614]: audit 2024-03-07T03:13:15.259917+0000 mon.a (mon.0) 1041 : audit 0 from='client.? 172.21.15.145:0/265997056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:17.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:16 smithi145 bash[21339]: cluster 2024-03-07T03:13:15.396976+0000 mgr.x (mgr.24511) 13 : cluster 0 pgmap v21: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 173 op/s 2024-03-07T03:13:17.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:16 smithi145 bash[21339]: cluster 2024-03-07T03:13:15.641024+0000 mon.a (mon.0) 1042 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2024-03-07T03:13:17.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:16 smithi145 bash[30906]: cluster 2024-03-07T03:13:15.396976+0000 mgr.x (mgr.24511) 13 : cluster 0 pgmap v21: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 173 op/s 2024-03-07T03:13:17.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:16 smithi145 bash[30906]: cluster 2024-03-07T03:13:15.641024+0000 mon.a (mon.0) 1042 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2024-03-07T03:13:17.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:16 smithi189 bash[23614]: cluster 2024-03-07T03:13:15.396976+0000 mgr.x (mgr.24511) 13 : cluster 0 pgmap v21: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 173 op/s 2024-03-07T03:13:17.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:16 smithi189 bash[23614]: cluster 2024-03-07T03:13:15.641024+0000 mon.a (mon.0) 1042 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2024-03-07T03:13:18.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:17 smithi145 bash[21339]: cluster 2024-03-07T03:13:16.638916+0000 mon.a (mon.0) 1043 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2024-03-07T03:13:18.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:17 smithi145 bash[21339]: audit 2024-03-07T03:13:16.928455+0000 mon.a (mon.0) 1044 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:17 smithi145 bash[21339]: audit 2024-03-07T03:13:16.938763+0000 mon.a (mon.0) 1045 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:17 smithi145 bash[21339]: audit 2024-03-07T03:13:17.383113+0000 mon.a (mon.0) 1046 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:17 smithi145 bash[21339]: audit 2024-03-07T03:13:17.396815+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:18.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:17 smithi145 bash[30906]: cluster 2024-03-07T03:13:16.638916+0000 mon.a (mon.0) 1043 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2024-03-07T03:13:18.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:17 smithi145 bash[30906]: audit 2024-03-07T03:13:16.928455+0000 mon.a (mon.0) 1044 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:18.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:17 smithi145 bash[30906]: audit 2024-03-07T03:13:16.938763+0000 mon.a (mon.0) 1045 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:18.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:17 smithi145 bash[30906]: audit 2024-03-07T03:13:17.383113+0000 mon.a (mon.0) 1046 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:18.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:17 smithi145 bash[30906]: audit 2024-03-07T03:13:17.396815+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:18.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:17 smithi189 bash[23614]: cluster 2024-03-07T03:13:16.638916+0000 mon.a (mon.0) 1043 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2024-03-07T03:13:18.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:17 smithi189 bash[23614]: audit 2024-03-07T03:13:16.928455+0000 mon.a (mon.0) 1044 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:18.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:17 smithi189 bash[23614]: audit 2024-03-07T03:13:16.938763+0000 mon.a (mon.0) 1045 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:18.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:17 smithi189 bash[23614]: audit 2024-03-07T03:13:17.383113+0000 mon.a (mon.0) 1046 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:18.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:17 smithi189 bash[23614]: audit 2024-03-07T03:13:17.396815+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:18.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[50256]: ts=2024-03-07T03:13:18.230Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-03-07T03:13:18.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[50256]: ts=2024-03-07T03:13:18.234Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-03-07T03:13:18.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[50256]: ts=2024-03-07T03:13:18.234Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-03-07T03:13:18.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[50256]: ts=2024-03-07T03:13:18.234Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-03-07T03:13:19.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: cluster 2024-03-07T03:13:17.397897+0000 mgr.x (mgr.24511) 14 : cluster 0 pgmap v24: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 173 op/s 2024-03-07T03:13:19.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: cluster 2024-03-07T03:13:17.662848+0000 mon.a (mon.0) 1048 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2024-03-07T03:13:19.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:17.684854+0000 mon.c (mon.2) 164 : audit 0 from='client.? 172.21.15.145:0/1180261309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:19.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:17.985332+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:18.000177+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:18.004350+0000 mon.b (mon.1) 58 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:18.006441+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:18.006485+0000 mon.b (mon.1) 59 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:18.008857+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:18.009013+0000 mon.b (mon.1) 60 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:18.011079+0000 mon.b (mon.1) 61 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:18.011109+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:18.013146+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:18.025386+0000 mon.a (mon.0) 1055 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: cluster 2024-03-07T03:13:18.453430+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2024-03-07T03:13:19.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:18.529823+0000 mon.b (mon.1) 62 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:19.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:18.624969+0000 mon.a (mon.0) 1057 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[21339]: audit 2024-03-07T03:13:18.639452+0000 mon.a (mon.0) 1058 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: cluster 2024-03-07T03:13:17.397897+0000 mgr.x (mgr.24511) 14 : cluster 0 pgmap v24: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 173 op/s 2024-03-07T03:13:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: cluster 2024-03-07T03:13:17.662848+0000 mon.a (mon.0) 1048 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2024-03-07T03:13:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:17.684854+0000 mon.c (mon.2) 164 : audit 0 from='client.? 172.21.15.145:0/1180261309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:17.985332+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:18.000177+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:18.004350+0000 mon.b (mon.1) 58 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:18.006441+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:18.006485+0000 mon.b (mon.1) 59 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:18.008857+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:18.009013+0000 mon.b (mon.1) 60 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:18.011079+0000 mon.b (mon.1) 61 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:18.011109+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:18.013146+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:18.025386+0000 mon.a (mon.0) 1055 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: cluster 2024-03-07T03:13:18.453430+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2024-03-07T03:13:19.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:18.529823+0000 mon.b (mon.1) 62 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:19.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:18.624969+0000 mon.a (mon.0) 1057 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:18 smithi145 bash[30906]: audit 2024-03-07T03:13:18.639452+0000 mon.a (mon.0) 1058 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: cluster 2024-03-07T03:13:17.397897+0000 mgr.x (mgr.24511) 14 : cluster 0 pgmap v24: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 173 op/s 2024-03-07T03:13:19.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: cluster 2024-03-07T03:13:17.662848+0000 mon.a (mon.0) 1048 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2024-03-07T03:13:19.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:17.684854+0000 mon.c (mon.2) 164 : audit 0 from='client.? 172.21.15.145:0/1180261309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:19.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:17.985332+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:18.000177+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:18.004350+0000 mon.b (mon.1) 58 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:18.006441+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:18.006485+0000 mon.b (mon.1) 59 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:18.008857+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:18.009013+0000 mon.b (mon.1) 60 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:18.011079+0000 mon.b (mon.1) 61 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:18.011109+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:18.013146+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:19.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:18.025386+0000 mon.a (mon.0) 1055 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: cluster 2024-03-07T03:13:18.453430+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2024-03-07T03:13:19.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:18.529823+0000 mon.b (mon.1) 62 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:19.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:18.624969+0000 mon.a (mon.0) 1057 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:18 smithi189 bash[23614]: audit 2024-03-07T03:13:18.639452+0000 mon.a (mon.0) 1058 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:19.323 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 0 2024-03-07T03:13:19.463 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:13:19.677 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:13:20.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:19 smithi145 bash[21339]: cephadm 2024-03-07T03:13:18.012875+0000 mgr.x (mgr.24511) 15 : cephadm 1 Adjusting osd_memory_target on smithi145 to 2524M 2024-03-07T03:13:20.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:19 smithi145 bash[30906]: cephadm 2024-03-07T03:13:18.012875+0000 mgr.x (mgr.24511) 15 : cephadm 1 Adjusting osd_memory_target on smithi145 to 2524M 2024-03-07T03:13:20.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:19 smithi189 bash[23614]: cephadm 2024-03-07T03:13:18.012875+0000 mgr.x (mgr.24511) 15 : cephadm 1 Adjusting osd_memory_target on smithi145 to 2524M 2024-03-07T03:13:21.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:20 smithi145 bash[21339]: cluster 2024-03-07T03:13:19.398825+0000 mgr.x (mgr.24511) 16 : cluster 0 pgmap v27: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:13:21.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:20 smithi145 bash[21339]: audit 2024-03-07T03:13:20.120905+0000 mon.c (mon.2) 165 : audit 0 from='client.? 172.21.15.145:0/1504935967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:21.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:20 smithi145 bash[30906]: cluster 2024-03-07T03:13:19.398825+0000 mgr.x (mgr.24511) 16 : cluster 0 pgmap v27: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:13:21.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:20 smithi145 bash[30906]: audit 2024-03-07T03:13:20.120905+0000 mon.c (mon.2) 165 : audit 0 from='client.? 172.21.15.145:0/1504935967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:21.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:20 smithi189 bash[23614]: cluster 2024-03-07T03:13:19.398825+0000 mgr.x (mgr.24511) 16 : cluster 0 pgmap v27: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:13:21.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:20 smithi189 bash[23614]: audit 2024-03-07T03:13:20.120905+0000 mon.c (mon.2) 165 : audit 0 from='client.? 172.21.15.145:0/1504935967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:22.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:21 smithi145 bash[21339]: cluster 2024-03-07T03:13:20.687467+0000 mon.a (mon.0) 1059 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2024-03-07T03:13:22.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:21 smithi145 bash[30906]: cluster 2024-03-07T03:13:20.687467+0000 mon.a (mon.0) 1059 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2024-03-07T03:13:22.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:21 smithi189 bash[23614]: cluster 2024-03-07T03:13:20.687467+0000 mon.a (mon.0) 1059 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2024-03-07T03:13:22.678 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:22 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:13:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-07T03:13:23.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:22 smithi145 bash[21339]: cluster 2024-03-07T03:13:21.399657+0000 mgr.x (mgr.24511) 17 : cluster 0 pgmap v29: 137 pgs: 137 active+clean; 581 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 3.4 MiB/s wr, 278 op/s 2024-03-07T03:13:23.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:22 smithi145 bash[21339]: cluster 2024-03-07T03:13:21.696338+0000 mon.a (mon.0) 1060 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2024-03-07T03:13:23.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:22 smithi145 bash[21339]: audit 2024-03-07T03:13:22.566350+0000 mon.a (mon.0) 1061 : audit 0 from='client.? 172.21.15.145:0/1369345429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:23.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:22 smithi145 bash[30906]: cluster 2024-03-07T03:13:21.399657+0000 mgr.x (mgr.24511) 17 : cluster 0 pgmap v29: 137 pgs: 137 active+clean; 581 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 3.4 MiB/s wr, 278 op/s 2024-03-07T03:13:23.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:22 smithi145 bash[30906]: cluster 2024-03-07T03:13:21.696338+0000 mon.a (mon.0) 1060 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2024-03-07T03:13:23.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:22 smithi145 bash[30906]: audit 2024-03-07T03:13:22.566350+0000 mon.a (mon.0) 1061 : audit 0 from='client.? 172.21.15.145:0/1369345429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:23.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:22 smithi189 bash[23614]: cluster 2024-03-07T03:13:21.399657+0000 mgr.x (mgr.24511) 17 : cluster 0 pgmap v29: 137 pgs: 137 active+clean; 581 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 3.4 MiB/s wr, 278 op/s 2024-03-07T03:13:23.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:22 smithi189 bash[23614]: cluster 2024-03-07T03:13:21.696338+0000 mon.a (mon.0) 1060 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2024-03-07T03:13:23.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:22 smithi189 bash[23614]: audit 2024-03-07T03:13:22.566350+0000 mon.a (mon.0) 1061 : audit 0 from='client.? 172.21.15.145:0/1369345429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:23.150 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.SnapsShrinkToZeroAfterPrepare (13361 ms) 2024-03-07T03:13:23.150 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare 2024-03-07T03:13:23.236 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:13:23.265 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:13:23.374 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:13:24.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:23 smithi145 bash[21339]: cluster 2024-03-07T03:13:22.703706+0000 mon.a (mon.0) 1062 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2024-03-07T03:13:24.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:23 smithi145 bash[30906]: cluster 2024-03-07T03:13:22.703706+0000 mon.a (mon.0) 1062 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2024-03-07T03:13:24.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:23 smithi189 bash[23614]: cluster 2024-03-07T03:13:22.703706+0000 mon.a (mon.0) 1062 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2024-03-07T03:13:24.720 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 8388609~8388609 2024-03-07T03:13:24.741 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 0~1000 2024-03-07T03:13:24.742 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 1010~1000 2024-03-07T03:13:24.742 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:13:25.037 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:24 smithi189 bash[23614]: cluster 2024-03-07T03:13:23.400414+0000 mgr.x (mgr.24511) 18 : cluster 0 pgmap v32: 137 pgs: 137 active+clean; 581 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 3.2 MiB/s wr, 267 op/s 2024-03-07T03:13:25.037 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:24 smithi189 bash[23614]: cluster 2024-03-07T03:13:23.713332+0000 mon.a (mon.0) 1063 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2024-03-07T03:13:25.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:24 smithi145 bash[21339]: cluster 2024-03-07T03:13:23.400414+0000 mgr.x (mgr.24511) 18 : cluster 0 pgmap v32: 137 pgs: 137 active+clean; 581 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 3.2 MiB/s wr, 267 op/s 2024-03-07T03:13:25.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:24 smithi145 bash[21339]: cluster 2024-03-07T03:13:23.713332+0000 mon.a (mon.0) 1063 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2024-03-07T03:13:25.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:24 smithi145 bash[30906]: cluster 2024-03-07T03:13:23.400414+0000 mgr.x (mgr.24511) 18 : cluster 0 pgmap v32: 137 pgs: 137 active+clean; 581 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 3.2 MiB/s wr, 267 op/s 2024-03-07T03:13:25.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:24 smithi145 bash[30906]: cluster 2024-03-07T03:13:23.713332+0000 mon.a (mon.0) 1063 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2024-03-07T03:13:25.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:25 smithi189 bash[23614]: cluster 2024-03-07T03:13:24.711298+0000 mon.a (mon.0) 1064 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2024-03-07T03:13:25.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:25 smithi189 bash[23614]: audit 2024-03-07T03:13:25.016958+0000 mon.c (mon.2) 166 : audit 0 from='client.? 172.21.15.145:0/514982217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:26.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:25 smithi145 bash[21339]: cluster 2024-03-07T03:13:24.711298+0000 mon.a (mon.0) 1064 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2024-03-07T03:13:26.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:25 smithi145 bash[21339]: audit 2024-03-07T03:13:25.016958+0000 mon.c (mon.2) 166 : audit 0 from='client.? 172.21.15.145:0/514982217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:26.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:25 smithi145 bash[30906]: cluster 2024-03-07T03:13:24.711298+0000 mon.a (mon.0) 1064 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2024-03-07T03:13:26.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:25 smithi145 bash[30906]: audit 2024-03-07T03:13:25.016958+0000 mon.c (mon.2) 166 : audit 0 from='client.? 172.21.15.145:0/514982217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:26.740 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 8388609~1000 2024-03-07T03:13:26.751 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 16777228~1000 2024-03-07T03:13:26.751 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 167772160 2024-03-07T03:13:26.771 INFO:tasks.workunit.client.0.smithi145.stdout:write: 2 83886079~8388609 2024-03-07T03:13:26.814 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:13:27.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:26 smithi145 bash[21339]: cluster 2024-03-07T03:13:25.401308+0000 mgr.x (mgr.24511) 19 : cluster 0 pgmap v35: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 317 op/s 2024-03-07T03:13:27.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:26 smithi145 bash[21339]: cluster 2024-03-07T03:13:25.733099+0000 mon.a (mon.0) 1065 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2024-03-07T03:13:27.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:26 smithi145 bash[30906]: cluster 2024-03-07T03:13:25.401308+0000 mgr.x (mgr.24511) 19 : cluster 0 pgmap v35: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 317 op/s 2024-03-07T03:13:27.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:26 smithi145 bash[30906]: cluster 2024-03-07T03:13:25.733099+0000 mon.a (mon.0) 1065 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2024-03-07T03:13:27.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:26 smithi189 bash[23614]: cluster 2024-03-07T03:13:25.401308+0000 mgr.x (mgr.24511) 19 : cluster 0 pgmap v35: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 317 op/s 2024-03-07T03:13:27.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:26 smithi189 bash[23614]: cluster 2024-03-07T03:13:25.733099+0000 mon.a (mon.0) 1065 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2024-03-07T03:13:28.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:27 smithi145 bash[21339]: cluster 2024-03-07T03:13:26.736235+0000 mon.a (mon.0) 1066 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2024-03-07T03:13:28.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:27 smithi145 bash[21339]: audit 2024-03-07T03:13:27.457297+0000 mon.a (mon.0) 1067 : audit 0 from='client.? 172.21.15.145:0/3997288528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:28.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:27 smithi145 bash[30906]: cluster 2024-03-07T03:13:26.736235+0000 mon.a (mon.0) 1066 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2024-03-07T03:13:28.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:27 smithi145 bash[30906]: audit 2024-03-07T03:13:27.457297+0000 mon.a (mon.0) 1067 : audit 0 from='client.? 172.21.15.145:0/3997288528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:28.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:27 smithi189 bash[23614]: cluster 2024-03-07T03:13:26.736235+0000 mon.a (mon.0) 1066 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2024-03-07T03:13:28.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:27 smithi189 bash[23614]: audit 2024-03-07T03:13:27.457297+0000 mon.a (mon.0) 1067 : audit 0 from='client.? 172.21.15.145:0/3997288528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:28.460 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:13:28.488 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 83886079~1 2024-03-07T03:13:28.497 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:13:29.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: cluster 2024-03-07T03:13:27.401802+0000 mgr.x (mgr.24511) 20 : cluster 0 pgmap v38: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 317 op/s 2024-03-07T03:13:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: cluster 2024-03-07T03:13:27.760139+0000 mon.a (mon.0) 1068 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2024-03-07T03:13:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: audit 2024-03-07T03:13:27.990442+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: audit 2024-03-07T03:13:28.008056+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: audit 2024-03-07T03:13:28.010637+0000 mon.b (mon.1) 63 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: audit 2024-03-07T03:13:28.013238+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: audit 2024-03-07T03:13:28.013485+0000 mon.b (mon.1) 64 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: audit 2024-03-07T03:13:28.016024+0000 mon.a (mon.0) 1072 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: audit 2024-03-07T03:13:28.016124+0000 mon.b (mon.1) 65 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: audit 2024-03-07T03:13:28.018667+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: audit 2024-03-07T03:13:28.018880+0000 mon.b (mon.1) 66 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: audit 2024-03-07T03:13:28.021439+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: audit 2024-03-07T03:13:28.032352+0000 mon.a (mon.0) 1075 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: audit 2024-03-07T03:13:28.033547+0000 mon.b (mon.1) 67 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:29.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: audit 2024-03-07T03:13:28.035138+0000 mon.b (mon.1) 68 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:29.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[21339]: cluster 2024-03-07T03:13:28.453554+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2024-03-07T03:13:29.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: cluster 2024-03-07T03:13:27.401802+0000 mgr.x (mgr.24511) 20 : cluster 0 pgmap v38: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 317 op/s 2024-03-07T03:13:29.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: cluster 2024-03-07T03:13:27.760139+0000 mon.a (mon.0) 1068 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2024-03-07T03:13:29.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: audit 2024-03-07T03:13:27.990442+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: audit 2024-03-07T03:13:28.008056+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: audit 2024-03-07T03:13:28.010637+0000 mon.b (mon.1) 63 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: audit 2024-03-07T03:13:28.013238+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: audit 2024-03-07T03:13:28.013485+0000 mon.b (mon.1) 64 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: audit 2024-03-07T03:13:28.016024+0000 mon.a (mon.0) 1072 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: audit 2024-03-07T03:13:28.016124+0000 mon.b (mon.1) 65 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: audit 2024-03-07T03:13:28.018667+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: audit 2024-03-07T03:13:28.018880+0000 mon.b (mon.1) 66 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: audit 2024-03-07T03:13:28.021439+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: audit 2024-03-07T03:13:28.032352+0000 mon.a (mon.0) 1075 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: audit 2024-03-07T03:13:28.033547+0000 mon.b (mon.1) 67 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:29.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: audit 2024-03-07T03:13:28.035138+0000 mon.b (mon.1) 68 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:29.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:28 smithi145 bash[30906]: cluster 2024-03-07T03:13:28.453554+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2024-03-07T03:13:29.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: cluster 2024-03-07T03:13:27.401802+0000 mgr.x (mgr.24511) 20 : cluster 0 pgmap v38: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 317 op/s 2024-03-07T03:13:29.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: cluster 2024-03-07T03:13:27.760139+0000 mon.a (mon.0) 1068 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2024-03-07T03:13:29.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: audit 2024-03-07T03:13:27.990442+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: audit 2024-03-07T03:13:28.008056+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: audit 2024-03-07T03:13:28.010637+0000 mon.b (mon.1) 63 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: audit 2024-03-07T03:13:28.013238+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: audit 2024-03-07T03:13:28.013485+0000 mon.b (mon.1) 64 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: audit 2024-03-07T03:13:28.016024+0000 mon.a (mon.0) 1072 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: audit 2024-03-07T03:13:28.016124+0000 mon.b (mon.1) 65 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: audit 2024-03-07T03:13:28.018667+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: audit 2024-03-07T03:13:28.018880+0000 mon.b (mon.1) 66 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: audit 2024-03-07T03:13:28.021439+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:13:29.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: audit 2024-03-07T03:13:28.032352+0000 mon.a (mon.0) 1075 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: audit 2024-03-07T03:13:28.033547+0000 mon.b (mon.1) 67 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:29.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: audit 2024-03-07T03:13:28.035138+0000 mon.b (mon.1) 68 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:29.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:28 smithi189 bash[23614]: cluster 2024-03-07T03:13:28.453554+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2024-03-07T03:13:29.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: cephadm 2024-03-07T03:13:28.021253+0000 mgr.x (mgr.24511) 21 : cephadm 1 Adjusting osd_memory_target on smithi189 to 2524M 2024-03-07T03:13:29.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: cephadm 2024-03-07T03:13:28.037701+0000 mgr.x (mgr.24511) 22 : cephadm 1 Updating smithi145:/etc/ceph/ceph.conf 2024-03-07T03:13:29.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: cephadm 2024-03-07T03:13:28.038084+0000 mgr.x (mgr.24511) 23 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T03:13:29.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: cephadm 2024-03-07T03:13:28.253830+0000 mgr.x (mgr.24511) 24 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:13:29.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: cephadm 2024-03-07T03:13:28.254099+0000 mgr.x (mgr.24511) 25 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:13:29.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: cephadm 2024-03-07T03:13:28.419110+0000 mgr.x (mgr.24511) 26 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:13:29.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: cephadm 2024-03-07T03:13:28.419444+0000 mgr.x (mgr.24511) 27 : cephadm 1 Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:13:29.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: cephadm 2024-03-07T03:13:28.583975+0000 mgr.x (mgr.24511) 28 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:13:29.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: cephadm 2024-03-07T03:13:28.586135+0000 mgr.x (mgr.24511) 29 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:13:29.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: audit 2024-03-07T03:13:28.762042+0000 mon.a (mon.0) 1077 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: audit 2024-03-07T03:13:28.767483+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: audit 2024-03-07T03:13:28.775036+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: audit 2024-03-07T03:13:28.786224+0000 mon.a (mon.0) 1080 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: audit 2024-03-07T03:13:28.794965+0000 mon.a (mon.0) 1081 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:29.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:29 smithi189 bash[23614]: cluster 2024-03-07T03:13:29.466526+0000 mon.a (mon.0) 1082 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2024-03-07T03:13:30.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: cephadm 2024-03-07T03:13:28.021253+0000 mgr.x (mgr.24511) 21 : cephadm 1 Adjusting osd_memory_target on smithi189 to 2524M 2024-03-07T03:13:30.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: cephadm 2024-03-07T03:13:28.037701+0000 mgr.x (mgr.24511) 22 : cephadm 1 Updating smithi145:/etc/ceph/ceph.conf 2024-03-07T03:13:30.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: cephadm 2024-03-07T03:13:28.038084+0000 mgr.x (mgr.24511) 23 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T03:13:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: cephadm 2024-03-07T03:13:28.253830+0000 mgr.x (mgr.24511) 24 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:13:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: cephadm 2024-03-07T03:13:28.254099+0000 mgr.x (mgr.24511) 25 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:13:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: cephadm 2024-03-07T03:13:28.419110+0000 mgr.x (mgr.24511) 26 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:13:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: cephadm 2024-03-07T03:13:28.419444+0000 mgr.x (mgr.24511) 27 : cephadm 1 Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:13:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: cephadm 2024-03-07T03:13:28.583975+0000 mgr.x (mgr.24511) 28 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:13:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: cephadm 2024-03-07T03:13:28.586135+0000 mgr.x (mgr.24511) 29 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:13:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: audit 2024-03-07T03:13:28.762042+0000 mon.a (mon.0) 1077 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: audit 2024-03-07T03:13:28.767483+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: audit 2024-03-07T03:13:28.775036+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: audit 2024-03-07T03:13:28.786224+0000 mon.a (mon.0) 1080 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: audit 2024-03-07T03:13:28.794965+0000 mon.a (mon.0) 1081 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[21339]: cluster 2024-03-07T03:13:29.466526+0000 mon.a (mon.0) 1082 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: cephadm 2024-03-07T03:13:28.021253+0000 mgr.x (mgr.24511) 21 : cephadm 1 Adjusting osd_memory_target on smithi189 to 2524M 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: cephadm 2024-03-07T03:13:28.037701+0000 mgr.x (mgr.24511) 22 : cephadm 1 Updating smithi145:/etc/ceph/ceph.conf 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: cephadm 2024-03-07T03:13:28.038084+0000 mgr.x (mgr.24511) 23 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: cephadm 2024-03-07T03:13:28.253830+0000 mgr.x (mgr.24511) 24 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: cephadm 2024-03-07T03:13:28.254099+0000 mgr.x (mgr.24511) 25 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: cephadm 2024-03-07T03:13:28.419110+0000 mgr.x (mgr.24511) 26 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: cephadm 2024-03-07T03:13:28.419444+0000 mgr.x (mgr.24511) 27 : cephadm 1 Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: cephadm 2024-03-07T03:13:28.583975+0000 mgr.x (mgr.24511) 28 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: cephadm 2024-03-07T03:13:28.586135+0000 mgr.x (mgr.24511) 29 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: audit 2024-03-07T03:13:28.762042+0000 mon.a (mon.0) 1077 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: audit 2024-03-07T03:13:28.767483+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: audit 2024-03-07T03:13:28.775036+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: audit 2024-03-07T03:13:28.786224+0000 mon.a (mon.0) 1080 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:30.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: audit 2024-03-07T03:13:28.794965+0000 mon.a (mon.0) 1081 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:30.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:29 smithi145 bash[30906]: cluster 2024-03-07T03:13:29.466526+0000 mon.a (mon.0) 1082 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2024-03-07T03:13:30.818 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:30 smithi189 bash[23614]: cephadm 2024-03-07T03:13:28.872969+0000 mgr.x (mgr.24511) 30 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:13:30.819 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:30 smithi189 bash[23614]: cephadm 2024-03-07T03:13:29.167942+0000 mgr.x (mgr.24511) 31 : cephadm 1 Reconfiguring daemon prometheus.a on smithi189 2024-03-07T03:13:30.819 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:30 smithi189 bash[23614]: cluster 2024-03-07T03:13:29.402499+0000 mgr.x (mgr.24511) 32 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:13:30.819 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:30 smithi189 bash[23614]: audit 2024-03-07T03:13:29.901899+0000 mon.c (mon.2) 167 : audit 0 from='client.? 172.21.15.145:0/2777241565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:30.819 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:30 smithi189 bash[23614]: cluster 2024-03-07T03:13:30.470488+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2024-03-07T03:13:31.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:30 smithi145 bash[21339]: cephadm 2024-03-07T03:13:28.872969+0000 mgr.x (mgr.24511) 30 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:13:31.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:30 smithi145 bash[21339]: cephadm 2024-03-07T03:13:29.167942+0000 mgr.x (mgr.24511) 31 : cephadm 1 Reconfiguring daemon prometheus.a on smithi189 2024-03-07T03:13:31.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:30 smithi145 bash[21339]: cluster 2024-03-07T03:13:29.402499+0000 mgr.x (mgr.24511) 32 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:13:31.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:30 smithi145 bash[21339]: audit 2024-03-07T03:13:29.901899+0000 mon.c (mon.2) 167 : audit 0 from='client.? 172.21.15.145:0/2777241565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:31.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:30 smithi145 bash[21339]: cluster 2024-03-07T03:13:30.470488+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2024-03-07T03:13:31.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:30 smithi145 bash[30906]: cephadm 2024-03-07T03:13:28.872969+0000 mgr.x (mgr.24511) 30 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:13:31.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:30 smithi145 bash[30906]: cephadm 2024-03-07T03:13:29.167942+0000 mgr.x (mgr.24511) 31 : cephadm 1 Reconfiguring daemon prometheus.a on smithi189 2024-03-07T03:13:31.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:30 smithi145 bash[30906]: cluster 2024-03-07T03:13:29.402499+0000 mgr.x (mgr.24511) 32 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:13:31.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:30 smithi145 bash[30906]: audit 2024-03-07T03:13:29.901899+0000 mon.c (mon.2) 167 : audit 0 from='client.? 172.21.15.145:0/2777241565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:31.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:30 smithi145 bash[30906]: cluster 2024-03-07T03:13:30.470488+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2024-03-07T03:13:32.290 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 167772160 2024-03-07T03:13:32.314 INFO:tasks.workunit.client.0.smithi145.stdout:write: * 83886080~1000 2024-03-07T03:13:32.374 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:13:32.613 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:13:32.751 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:32 smithi189 systemd[1]: Stopping Ceph prometheus.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:13:32.751 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[23614]: cluster 2024-03-07T03:13:31.403266+0000 mgr.x (mgr.24511) 33 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 142 KiB/s rd, 8.0 MiB/s wr, 203 op/s 2024-03-07T03:13:32.752 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[23614]: cluster 2024-03-07T03:13:31.473279+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2024-03-07T03:13:32.752 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[23614]: audit 2024-03-07T03:13:32.375542+0000 mon.a (mon.0) 1085 : audit 0 from='client.? 172.21.15.145:0/2533984674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:32.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:32 smithi145 bash[21623]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:13:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-07T03:13:32.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:32 smithi145 bash[21339]: cluster 2024-03-07T03:13:31.403266+0000 mgr.x (mgr.24511) 33 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 142 KiB/s rd, 8.0 MiB/s wr, 203 op/s 2024-03-07T03:13:32.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:32 smithi145 bash[21339]: cluster 2024-03-07T03:13:31.473279+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2024-03-07T03:13:32.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:32 smithi145 bash[21339]: audit 2024-03-07T03:13:32.375542+0000 mon.a (mon.0) 1085 : audit 0 from='client.? 172.21.15.145:0/2533984674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:32.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:32 smithi145 bash[30906]: cluster 2024-03-07T03:13:31.403266+0000 mgr.x (mgr.24511) 33 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 142 KiB/s rd, 8.0 MiB/s wr, 203 op/s 2024-03-07T03:13:32.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:32 smithi145 bash[30906]: cluster 2024-03-07T03:13:31.473279+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2024-03-07T03:13:32.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:32 smithi145 bash[30906]: audit 2024-03-07T03:13:32.375542+0000 mon.a (mon.0) 1085 : audit 0 from='client.? 172.21.15.145:0/2533984674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:33.081 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[50256]: ts=2024-03-07T03:13:32.762Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-07T03:13:33.081 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[50256]: ts=2024-03-07T03:13:32.762Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-07T03:13:33.081 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[50256]: ts=2024-03-07T03:13:32.762Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-07T03:13:33.081 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[50256]: ts=2024-03-07T03:13:32.762Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-07T03:13:33.081 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[50256]: ts=2024-03-07T03:13:32.762Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-07T03:13:33.082 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[50256]: ts=2024-03-07T03:13:32.762Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-07T03:13:33.082 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[50256]: ts=2024-03-07T03:13:32.762Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-07T03:13:33.082 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[50256]: ts=2024-03-07T03:13:32.762Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-07T03:13:33.082 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[50256]: ts=2024-03-07T03:13:32.762Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-07T03:13:33.082 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[50256]: ts=2024-03-07T03:13:32.762Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-07T03:13:33.082 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[50256]: ts=2024-03-07T03:13:32.762Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-07T03:13:33.082 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:32 smithi189 bash[50256]: ts=2024-03-07T03:13:32.763Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-07T03:13:33.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:33 smithi189 bash[58402]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-prometheus-a 2024-03-07T03:13:33.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:33 smithi189 bash[58463]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-prometheus-a 2024-03-07T03:13:33.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:33 smithi189 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-07T03:13:33.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:33 smithi189 systemd[1]: Stopped Ceph prometheus.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:13:33.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:33 smithi189 systemd[1]: Started Ceph prometheus.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:13:34.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[23614]: cluster 2024-03-07T03:13:33.404058+0000 mgr.x (mgr.24511) 34 : cluster 0 pgmap v46: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 114 KiB/s rd, 6.5 MiB/s wr, 164 op/s 2024-03-07T03:13:34.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[23614]: cluster 2024-03-07T03:13:33.480349+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2024-03-07T03:13:34.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[23614]: audit 2024-03-07T03:13:33.528437+0000 mon.b (mon.1) 69 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:34.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[23614]: audit 2024-03-07T03:13:33.538414+0000 mon.a (mon.0) 1087 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:34.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[23614]: audit 2024-03-07T03:13:33.554301+0000 mon.a (mon.0) 1088 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:34.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[23614]: audit 2024-03-07T03:13:33.561404+0000 mon.b (mon.1) 70 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:13:34.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[23614]: audit 2024-03-07T03:13:33.561917+0000 mgr.x (mgr.24511) 35 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:13:34.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[23614]: audit 2024-03-07T03:13:33.568669+0000 mon.b (mon.1) 71 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:34.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[23614]: cephadm 2024-03-07T03:13:33.569691+0000 mgr.x (mgr.24511) 36 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-03-07T03:13:34.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[23614]: cephadm 2024-03-07T03:13:33.570250+0000 mgr.x (mgr.24511) 37 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-03-07T03:13:34.582 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.412Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-07T03:13:34.582 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.412Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-07T03:13:34.582 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.412Z caller=main.go:570 level=info host_details="(Linux 5.15.0-97-generic #107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024 x86_64 smithi189 (none))" 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.413Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.413Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.415Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.416Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.417Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.417Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.418Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.418Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.409µs 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.418Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.421Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.431Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.431Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.432Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=40.359µs wal_replay_duration=13.021152ms wbl_replay_duration=196ns total_replay_duration=13.093501ms 2024-03-07T03:13:34.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.433Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-07T03:13:34.584 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.433Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-07T03:13:34.584 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.433Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-07T03:13:34.584 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.457Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.82996ms db_storage=1.061µs remote_storage=1.423µs web_handler=448ns query_engine=702ns scrape=261.49µs scrape_sd=113.668µs notify=11.337µs notify_sd=29.421µs rules=22.998193ms tracing=6.794µs 2024-03-07T03:13:34.584 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.457Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-07T03:13:34.584 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:34 smithi189 bash[58488]: ts=2024-03-07T03:13:34.457Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-07T03:13:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[30906]: cluster 2024-03-07T03:13:33.404058+0000 mgr.x (mgr.24511) 34 : cluster 0 pgmap v46: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 114 KiB/s rd, 6.5 MiB/s wr, 164 op/s 2024-03-07T03:13:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[30906]: cluster 2024-03-07T03:13:33.480349+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2024-03-07T03:13:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[30906]: audit 2024-03-07T03:13:33.528437+0000 mon.b (mon.1) 69 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[30906]: audit 2024-03-07T03:13:33.538414+0000 mon.a (mon.0) 1087 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[30906]: audit 2024-03-07T03:13:33.554301+0000 mon.a (mon.0) 1088 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[30906]: audit 2024-03-07T03:13:33.561404+0000 mon.b (mon.1) 70 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:13:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[30906]: audit 2024-03-07T03:13:33.561917+0000 mgr.x (mgr.24511) 35 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:13:34.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[30906]: audit 2024-03-07T03:13:33.568669+0000 mon.b (mon.1) 71 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:34.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[30906]: cephadm 2024-03-07T03:13:33.569691+0000 mgr.x (mgr.24511) 36 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-03-07T03:13:34.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[30906]: cephadm 2024-03-07T03:13:33.570250+0000 mgr.x (mgr.24511) 37 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-03-07T03:13:34.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[21339]: cluster 2024-03-07T03:13:33.404058+0000 mgr.x (mgr.24511) 34 : cluster 0 pgmap v46: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 114 KiB/s rd, 6.5 MiB/s wr, 164 op/s 2024-03-07T03:13:34.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[21339]: cluster 2024-03-07T03:13:33.480349+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2024-03-07T03:13:34.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[21339]: audit 2024-03-07T03:13:33.528437+0000 mon.b (mon.1) 69 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:34.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[21339]: audit 2024-03-07T03:13:33.538414+0000 mon.a (mon.0) 1087 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:34.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[21339]: audit 2024-03-07T03:13:33.554301+0000 mon.a (mon.0) 1088 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:34.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[21339]: audit 2024-03-07T03:13:33.561404+0000 mon.b (mon.1) 70 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:13:34.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[21339]: audit 2024-03-07T03:13:33.561917+0000 mgr.x (mgr.24511) 35 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:13:34.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[21339]: audit 2024-03-07T03:13:33.568669+0000 mon.b (mon.1) 71 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:34.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[21339]: cephadm 2024-03-07T03:13:33.569691+0000 mgr.x (mgr.24511) 36 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-03-07T03:13:34.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:34 smithi145 bash[21339]: cephadm 2024-03-07T03:13:33.570250+0000 mgr.x (mgr.24511) 37 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-03-07T03:13:35.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:35 smithi145 bash[21339]: cluster 2024-03-07T03:13:34.479569+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2024-03-07T03:13:35.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:35 smithi145 bash[21339]: audit 2024-03-07T03:13:34.826174+0000 mon.c (mon.2) 168 : audit 0 from='client.? 172.21.15.145:0/587830426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:35.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:35 smithi145 bash[30906]: cluster 2024-03-07T03:13:34.479569+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2024-03-07T03:13:35.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:35 smithi145 bash[30906]: audit 2024-03-07T03:13:34.826174+0000 mon.c (mon.2) 168 : audit 0 from='client.? 172.21.15.145:0/587830426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:35.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:35 smithi189 bash[23614]: cluster 2024-03-07T03:13:34.479569+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2024-03-07T03:13:35.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:35 smithi189 bash[23614]: audit 2024-03-07T03:13:34.826174+0000 mon.c (mon.2) 168 : audit 0 from='client.? 172.21.15.145:0/587830426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:36.164 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare (13014 ms) 2024-03-07T03:13:36.164 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare2 2024-03-07T03:13:36.253 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:13:36.278 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 41943040~10 2024-03-07T03:13:36.300 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:13:36.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[21339]: cluster 2024-03-07T03:13:35.405703+0000 mgr.x (mgr.24511) 38 : cluster 0 pgmap v49: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 4.9 MiB/s wr, 343 op/s 2024-03-07T03:13:36.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[21339]: cluster 2024-03-07T03:13:35.506863+0000 mon.a (mon.0) 1090 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2024-03-07T03:13:36.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[21339]: audit 2024-03-07T03:13:36.204174+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:36.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[21339]: audit 2024-03-07T03:13:36.204758+0000 mon.b (mon.1) 72 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:13:36.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[21339]: audit 2024-03-07T03:13:36.207396+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:13:36.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[21339]: audit 2024-03-07T03:13:36.208027+0000 mon.b (mon.1) 73 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:13:36.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[21339]: audit 2024-03-07T03:13:36.210225+0000 mon.b (mon.1) 74 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:36.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[30906]: cluster 2024-03-07T03:13:35.405703+0000 mgr.x (mgr.24511) 38 : cluster 0 pgmap v49: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 4.9 MiB/s wr, 343 op/s 2024-03-07T03:13:36.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[30906]: cluster 2024-03-07T03:13:35.506863+0000 mon.a (mon.0) 1090 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2024-03-07T03:13:36.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[30906]: audit 2024-03-07T03:13:36.204174+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:36.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[30906]: audit 2024-03-07T03:13:36.204758+0000 mon.b (mon.1) 72 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:13:36.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[30906]: audit 2024-03-07T03:13:36.207396+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:13:36.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[30906]: audit 2024-03-07T03:13:36.208027+0000 mon.b (mon.1) 73 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:13:36.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:36 smithi145 bash[30906]: audit 2024-03-07T03:13:36.210225+0000 mon.b (mon.1) 74 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:36 smithi189 bash[23614]: cluster 2024-03-07T03:13:35.405703+0000 mgr.x (mgr.24511) 38 : cluster 0 pgmap v49: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 4.9 MiB/s wr, 343 op/s 2024-03-07T03:13:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:36 smithi189 bash[23614]: cluster 2024-03-07T03:13:35.506863+0000 mon.a (mon.0) 1090 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2024-03-07T03:13:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:36 smithi189 bash[23614]: audit 2024-03-07T03:13:36.204174+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:36 smithi189 bash[23614]: audit 2024-03-07T03:13:36.204758+0000 mon.b (mon.1) 72 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:13:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:36 smithi189 bash[23614]: audit 2024-03-07T03:13:36.207396+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:13:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:36 smithi189 bash[23614]: audit 2024-03-07T03:13:36.208027+0000 mon.b (mon.1) 73 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:13:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:36 smithi189 bash[23614]: audit 2024-03-07T03:13:36.210225+0000 mon.b (mon.1) 74 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:37.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:37 smithi145 bash[21339]: cephadm 2024-03-07T03:13:36.190402+0000 mgr.x (mgr.24511) 39 : cephadm 1 Upgrade: Updating mgr.y 2024-03-07T03:13:37.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:37 smithi145 bash[21339]: cephadm 2024-03-07T03:13:36.212543+0000 mgr.x (mgr.24511) 40 : cephadm 1 Deploying daemon mgr.y on smithi145 2024-03-07T03:13:37.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:37 smithi145 bash[21339]: cluster 2024-03-07T03:13:37.206602+0000 mon.a (mon.0) 1093 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2024-03-07T03:13:37.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:37 smithi145 bash[21339]: audit 2024-03-07T03:13:37.274776+0000 mon.a (mon.0) 1094 : audit 0 from='client.? 172.21.15.145:0/2314354675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:37.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:37 smithi145 bash[30906]: cephadm 2024-03-07T03:13:36.190402+0000 mgr.x (mgr.24511) 39 : cephadm 1 Upgrade: Updating mgr.y 2024-03-07T03:13:37.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:37 smithi145 bash[30906]: cephadm 2024-03-07T03:13:36.212543+0000 mgr.x (mgr.24511) 40 : cephadm 1 Deploying daemon mgr.y on smithi145 2024-03-07T03:13:37.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:37 smithi145 bash[30906]: cluster 2024-03-07T03:13:37.206602+0000 mon.a (mon.0) 1093 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2024-03-07T03:13:37.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:37 smithi145 bash[30906]: audit 2024-03-07T03:13:37.274776+0000 mon.a (mon.0) 1094 : audit 0 from='client.? 172.21.15.145:0/2314354675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:37.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:37 smithi189 bash[23614]: cephadm 2024-03-07T03:13:36.190402+0000 mgr.x (mgr.24511) 39 : cephadm 1 Upgrade: Updating mgr.y 2024-03-07T03:13:37.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:37 smithi189 bash[23614]: cephadm 2024-03-07T03:13:36.212543+0000 mgr.x (mgr.24511) 40 : cephadm 1 Deploying daemon mgr.y on smithi145 2024-03-07T03:13:37.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:37 smithi189 bash[23614]: cluster 2024-03-07T03:13:37.206602+0000 mon.a (mon.0) 1093 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2024-03-07T03:13:37.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:37 smithi189 bash[23614]: audit 2024-03-07T03:13:37.274776+0000 mon.a (mon.0) 1094 : audit 0 from='client.? 172.21.15.145:0/2314354675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:38.224 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 41943040 2024-03-07T03:13:38.271 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:13:38.739 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:38.739 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:13:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:38.740 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:13:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:38.740 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:13:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:38.740 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:38 smithi145 bash[21339]: cluster 2024-03-07T03:13:37.406439+0000 mgr.x (mgr.24511) 41 : cluster 0 pgmap v52: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 423 op/s 2024-03-07T03:13:38.740 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:38 smithi145 bash[21339]: cluster 2024-03-07T03:13:38.209351+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2024-03-07T03:13:38.740 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:38 smithi145 bash[21339]: cluster 2024-03-07T03:13:38.461338+0000 mon.a (mon.0) 1096 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2024-03-07T03:13:38.740 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:38.741 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:38 smithi145 bash[30906]: cluster 2024-03-07T03:13:37.406439+0000 mgr.x (mgr.24511) 41 : cluster 0 pgmap v52: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 423 op/s 2024-03-07T03:13:38.741 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:38 smithi145 bash[30906]: cluster 2024-03-07T03:13:38.209351+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2024-03-07T03:13:38.741 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:38 smithi145 bash[30906]: cluster 2024-03-07T03:13:38.461338+0000 mon.a (mon.0) 1096 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2024-03-07T03:13:38.741 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:38.741 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:13:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:38.741 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:13:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:38.742 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:13:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:38.753 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:13:38.770 INFO:tasks.workunit.client.0.smithi145.stdout:write: Y 41943040~5 2024-03-07T03:13:38.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:38 smithi189 bash[23614]: cluster 2024-03-07T03:13:37.406439+0000 mgr.x (mgr.24511) 41 : cluster 0 pgmap v52: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 423 op/s 2024-03-07T03:13:38.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:38 smithi189 bash[23614]: cluster 2024-03-07T03:13:38.209351+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2024-03-07T03:13:38.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:38 smithi189 bash[23614]: cluster 2024-03-07T03:13:38.461338+0000 mon.a (mon.0) 1096 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2024-03-07T03:13:38.938 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:13:38.990 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:38 smithi145 systemd[1]: Stopping Ceph mgr.y for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:13:39.059 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:13:39.681 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare2 (3517 ms) 2024-03-07T03:13:39.681 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.SnapsSnapAfterPrepare 2024-03-07T03:13:39.736 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:13:39.758 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:13:39.842 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:13:39.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:39 smithi145 bash[72095]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mgr-y 2024-03-07T03:13:39.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:39 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-03-07T03:13:39.888 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:39 smithi145 bash[72181]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mgr-y 2024-03-07T03:13:39.889 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:39 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.y.service: Failed with result 'exit-code'. 2024-03-07T03:13:39.889 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:39 smithi145 systemd[1]: Stopped Ceph mgr.y for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:13:40.143 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:40 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:40.143 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:40 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:40.144 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:40 smithi145 systemd[1]: Started Ceph mgr.y for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:13:40.144 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:13:39 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:40.145 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:13:39 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:40.145 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:13:40 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:40.145 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:40 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:40.145 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:13:39 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:40.146 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:13:40 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:40.146 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:13:39 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:13:40.519 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:40 smithi189 bash[52103]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:13:40] "GET /metrics HTTP/1.1" 200 38627 "" "Prometheus/2.43.0" 2024-03-07T03:13:40.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:40 smithi145 bash[21339]: cluster 2024-03-07T03:13:39.407210+0000 mgr.x (mgr.24511) 42 : cluster 0 pgmap v55: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:13:40.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:40 smithi145 bash[21339]: cluster 2024-03-07T03:13:39.514729+0000 mon.a (mon.0) 1097 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2024-03-07T03:13:40.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:40 smithi145 bash[21339]: audit 2024-03-07T03:13:39.750115+0000 mon.a (mon.0) 1098 : audit 0 from='client.? 172.21.15.145:0/3615081961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:40.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:40 smithi145 bash[21339]: audit 2024-03-07T03:13:40.163415+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:40.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:40 smithi145 bash[21339]: audit 2024-03-07T03:13:40.176452+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:40.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:40 smithi145 bash[21339]: audit 2024-03-07T03:13:40.185935+0000 mon.b (mon.1) 75 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:40.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:40 smithi145 bash[30906]: cluster 2024-03-07T03:13:39.407210+0000 mgr.x (mgr.24511) 42 : cluster 0 pgmap v55: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:13:40.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:40 smithi145 bash[30906]: cluster 2024-03-07T03:13:39.514729+0000 mon.a (mon.0) 1097 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2024-03-07T03:13:40.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:40 smithi145 bash[30906]: audit 2024-03-07T03:13:39.750115+0000 mon.a (mon.0) 1098 : audit 0 from='client.? 172.21.15.145:0/3615081961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:40.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:40 smithi145 bash[30906]: audit 2024-03-07T03:13:40.163415+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:40.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:40 smithi145 bash[30906]: audit 2024-03-07T03:13:40.176452+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:40.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:40 smithi145 bash[30906]: audit 2024-03-07T03:13:40.185935+0000 mon.b (mon.1) 75 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:40.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:40 smithi189 bash[23614]: cluster 2024-03-07T03:13:39.407210+0000 mgr.x (mgr.24511) 42 : cluster 0 pgmap v55: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:13:40.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:40 smithi189 bash[23614]: cluster 2024-03-07T03:13:39.514729+0000 mon.a (mon.0) 1097 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2024-03-07T03:13:40.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:40 smithi189 bash[23614]: audit 2024-03-07T03:13:39.750115+0000 mon.a (mon.0) 1098 : audit 0 from='client.? 172.21.15.145:0/3615081961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:40.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:40 smithi189 bash[23614]: audit 2024-03-07T03:13:40.163415+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:40.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:40 smithi189 bash[23614]: audit 2024-03-07T03:13:40.176452+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:40.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:40 smithi189 bash[23614]: audit 2024-03-07T03:13:40.185935+0000 mon.b (mon.1) 75 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:41.552 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 8388609~8388609 2024-03-07T03:13:41.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:41 smithi145 bash[21339]: cluster 2024-03-07T03:13:40.519808+0000 mon.a (mon.0) 1101 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2024-03-07T03:13:41.572 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:41 smithi145 bash[72238]: debug 2024-03-07T03:13:41.425+0000 7ff4851ce700 1 -- 172.21.15.145:0/2549376795 <== mon.2 v2:172.21.15.145:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5575d1ce65a0 con 0x5575d2a6a800 2024-03-07T03:13:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:41 smithi145 bash[30906]: cluster 2024-03-07T03:13:40.519808+0000 mon.a (mon.0) 1101 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2024-03-07T03:13:41.591 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 0~1000 2024-03-07T03:13:41.591 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 1010~1000 2024-03-07T03:13:41.591 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:13:41.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:41 smithi189 bash[23614]: cluster 2024-03-07T03:13:40.519808+0000 mon.a (mon.0) 1101 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2024-03-07T03:13:42.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:41 smithi145 bash[72238]: debug 2024-03-07T03:13:41.581+0000 7ff48f3d0200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:13:42.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:42 smithi145 bash[21339]: cluster 2024-03-07T03:13:41.408011+0000 mgr.x (mgr.24511) 43 : cluster 0 pgmap v58: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 22 KiB/s wr, 417 op/s 2024-03-07T03:13:42.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:42 smithi145 bash[21339]: cluster 2024-03-07T03:13:41.544389+0000 mon.a (mon.0) 1102 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2024-03-07T03:13:42.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:42 smithi145 bash[21339]: audit 2024-03-07T03:13:42.210025+0000 mon.a (mon.0) 1103 : audit 0 from='client.? 172.21.15.145:0/553621884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:42.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:42 smithi145 bash[30906]: cluster 2024-03-07T03:13:41.408011+0000 mgr.x (mgr.24511) 43 : cluster 0 pgmap v58: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 22 KiB/s wr, 417 op/s 2024-03-07T03:13:42.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:42 smithi145 bash[30906]: cluster 2024-03-07T03:13:41.544389+0000 mon.a (mon.0) 1102 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2024-03-07T03:13:42.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:42 smithi145 bash[30906]: audit 2024-03-07T03:13:42.210025+0000 mon.a (mon.0) 1103 : audit 0 from='client.? 172.21.15.145:0/553621884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:42.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:42 smithi189 bash[23614]: cluster 2024-03-07T03:13:41.408011+0000 mgr.x (mgr.24511) 43 : cluster 0 pgmap v58: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 22 KiB/s wr, 417 op/s 2024-03-07T03:13:42.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:42 smithi189 bash[23614]: cluster 2024-03-07T03:13:41.544389+0000 mon.a (mon.0) 1102 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2024-03-07T03:13:42.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:42 smithi189 bash[23614]: audit 2024-03-07T03:13:42.210025+0000 mon.a (mon.0) 1103 : audit 0 from='client.? 172.21.15.145:0/553621884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:43.484 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 8388609~1000 2024-03-07T03:13:43.500 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 16777228~1000 2024-03-07T03:13:43.500 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 167772160 2024-03-07T03:13:43.505 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:43 smithi145 bash[72238]: debug 2024-03-07T03:13:43.245+0000 7ff48f3d0200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:13:43.506 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:43 smithi145 bash[72238]: debug 2024-03-07T03:13:43.341+0000 7ff48f3d0200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:13:43.523 INFO:tasks.workunit.client.0.smithi145.stdout:write: 2 83886079~8388609 2024-03-07T03:13:43.579 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:13:43.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:43 smithi145 bash[21339]: cluster 2024-03-07T03:13:42.552721+0000 mon.a (mon.0) 1104 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2024-03-07T03:13:43.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:43 smithi145 bash[21339]: cluster 2024-03-07T03:13:43.462884+0000 mon.a (mon.0) 1105 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2024-03-07T03:13:43.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:43 smithi145 bash[72238]: debug 2024-03-07T03:13:43.625+0000 7ff48f3d0200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:13:43.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:43 smithi145 bash[30906]: cluster 2024-03-07T03:13:42.552721+0000 mon.a (mon.0) 1104 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2024-03-07T03:13:43.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:43 smithi145 bash[30906]: cluster 2024-03-07T03:13:43.462884+0000 mon.a (mon.0) 1105 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2024-03-07T03:13:43.810 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:13:43 smithi145 bash[57902]: ts=2024-03-07T03:13:43.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:13:43.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:43 smithi189 bash[23614]: cluster 2024-03-07T03:13:42.552721+0000 mon.a (mon.0) 1104 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2024-03-07T03:13:43.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:43 smithi189 bash[23614]: cluster 2024-03-07T03:13:43.462884+0000 mon.a (mon.0) 1105 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2024-03-07T03:13:44.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:44 smithi145 bash[30906]: cluster 2024-03-07T03:13:43.408779+0000 mgr.x (mgr.24511) 44 : cluster 0 pgmap v61: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 22 KiB/s wr, 417 op/s 2024-03-07T03:13:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:44 smithi145 bash[21339]: cluster 2024-03-07T03:13:43.408779+0000 mgr.x (mgr.24511) 44 : cluster 0 pgmap v61: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 22 KiB/s wr, 417 op/s 2024-03-07T03:13:44.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:44 smithi145 bash[72238]: debug 2024-03-07T03:13:44.445+0000 7ff48f3d0200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:13:44.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:44 smithi145 bash[72238]: debug 2024-03-07T03:13:44.557+0000 7ff48f3d0200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:13:44.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:44 smithi145 bash[72238]: debug 2024-03-07T03:13:44.673+0000 7ff48f3d0200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:13:44.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:44 smithi189 bash[23614]: cluster 2024-03-07T03:13:43.408779+0000 mgr.x (mgr.24511) 44 : cluster 0 pgmap v61: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 22 KiB/s wr, 417 op/s 2024-03-07T03:13:45.308 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:45 smithi145 bash[72238]: debug 2024-03-07T03:13:45.033+0000 7ff48f3d0200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:13:45.308 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:45 smithi145 bash[72238]: debug 2024-03-07T03:13:45.149+0000 7ff48f3d0200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:13:45.309 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:45 smithi145 bash[72238]: debug 2024-03-07T03:13:45.261+0000 7ff48f3d0200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:13:45.584 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:13:45.619 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 83886079~1 2024-03-07T03:13:45.630 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:13:45.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:45 smithi189 bash[23614]: cluster 2024-03-07T03:13:44.573973+0000 mon.a (mon.0) 1106 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2024-03-07T03:13:45.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:45 smithi189 bash[23614]: audit 2024-03-07T03:13:44.642251+0000 mon.a (mon.0) 1107 : audit 0 from='client.? 172.21.15.145:0/1001953510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:45.935 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:45 smithi145 bash[72238]: debug 2024-03-07T03:13:45.661+0000 7ff48f3d0200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:13:45.935 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:45 smithi145 bash[21339]: cluster 2024-03-07T03:13:44.573973+0000 mon.a (mon.0) 1106 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2024-03-07T03:13:45.935 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:45 smithi145 bash[21339]: audit 2024-03-07T03:13:44.642251+0000 mon.a (mon.0) 1107 : audit 0 from='client.? 172.21.15.145:0/1001953510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:45.935 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:45 smithi145 bash[30906]: cluster 2024-03-07T03:13:44.573973+0000 mon.a (mon.0) 1106 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2024-03-07T03:13:45.935 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:45 smithi145 bash[30906]: audit 2024-03-07T03:13:44.642251+0000 mon.a (mon.0) 1107 : audit 0 from='client.? 172.21.15.145:0/1001953510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:46.212 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:45 smithi145 bash[72238]: debug 2024-03-07T03:13:45.945+0000 7ff48f3d0200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:13:46.558 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[72238]: debug 2024-03-07T03:13:46.221+0000 7ff48f3d0200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:13:46.558 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[72238]: debug 2024-03-07T03:13:46.345+0000 7ff48f3d0200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:13:46.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:46 smithi189 bash[23614]: cluster 2024-03-07T03:13:45.409591+0000 mgr.x (mgr.24511) 45 : cluster 0 pgmap v64: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 12 MiB/s wr, 161 op/s 2024-03-07T03:13:46.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:46 smithi189 bash[23614]: cluster 2024-03-07T03:13:45.578868+0000 mon.a (mon.0) 1108 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2024-03-07T03:13:46.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:46 smithi189 bash[23614]: audit 2024-03-07T03:13:45.978312+0000 mon.a (mon.0) 1109 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:46.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:46 smithi189 bash[23614]: audit 2024-03-07T03:13:45.987759+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:46.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:46 smithi189 bash[23614]: audit 2024-03-07T03:13:46.333319+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:46.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:46 smithi189 bash[23614]: audit 2024-03-07T03:13:46.344388+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:46.899 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[72238]: debug 2024-03-07T03:13:46.641+0000 7ff48f3d0200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:13:46.899 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[21339]: cluster 2024-03-07T03:13:45.409591+0000 mgr.x (mgr.24511) 45 : cluster 0 pgmap v64: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 12 MiB/s wr, 161 op/s 2024-03-07T03:13:46.899 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[21339]: cluster 2024-03-07T03:13:45.578868+0000 mon.a (mon.0) 1108 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2024-03-07T03:13:46.899 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[21339]: audit 2024-03-07T03:13:45.978312+0000 mon.a (mon.0) 1109 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:46.899 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[21339]: audit 2024-03-07T03:13:45.987759+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:46.899 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[21339]: audit 2024-03-07T03:13:46.333319+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:46.899 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[21339]: audit 2024-03-07T03:13:46.344388+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:46.900 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[30906]: cluster 2024-03-07T03:13:45.409591+0000 mgr.x (mgr.24511) 45 : cluster 0 pgmap v64: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 12 MiB/s wr, 161 op/s 2024-03-07T03:13:46.900 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[30906]: cluster 2024-03-07T03:13:45.578868+0000 mon.a (mon.0) 1108 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2024-03-07T03:13:46.900 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[30906]: audit 2024-03-07T03:13:45.978312+0000 mon.a (mon.0) 1109 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:46.900 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[30906]: audit 2024-03-07T03:13:45.987759+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:46.900 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[30906]: audit 2024-03-07T03:13:46.333319+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:46.900 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[30906]: audit 2024-03-07T03:13:46.344388+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:47.308 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:46 smithi145 bash[72238]: debug 2024-03-07T03:13:46.909+0000 7ff48f3d0200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:13:47.719 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:47 smithi145 bash[72238]: debug 2024-03-07T03:13:47.345+0000 7ff48f3d0200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:13:47.719 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:47 smithi145 bash[72238]: debug 2024-03-07T03:13:47.469+0000 7ff48f3d0200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:13:47.720 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:47 smithi145 bash[21339]: cluster 2024-03-07T03:13:46.579205+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2024-03-07T03:13:47.720 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:47 smithi145 bash[21339]: audit 2024-03-07T03:13:47.058430+0000 mon.c (mon.2) 169 : audit 0 from='client.? 172.21.15.145:0/2790718999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:47.720 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:47 smithi145 bash[21339]: audit 2024-03-07T03:13:47.088143+0000 mon.a (mon.0) 1114 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:47.720 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:47 smithi145 bash[21339]: audit 2024-03-07T03:13:47.103168+0000 mon.a (mon.0) 1115 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:47.720 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:47 smithi145 bash[30906]: cluster 2024-03-07T03:13:46.579205+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2024-03-07T03:13:47.720 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:47 smithi145 bash[30906]: audit 2024-03-07T03:13:47.058430+0000 mon.c (mon.2) 169 : audit 0 from='client.? 172.21.15.145:0/2790718999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:47.720 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:47 smithi145 bash[30906]: audit 2024-03-07T03:13:47.088143+0000 mon.a (mon.0) 1114 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:47.720 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:47 smithi145 bash[30906]: audit 2024-03-07T03:13:47.103168+0000 mon.a (mon.0) 1115 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:47.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:47 smithi189 bash[23614]: cluster 2024-03-07T03:13:46.579205+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2024-03-07T03:13:47.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:47 smithi189 bash[23614]: audit 2024-03-07T03:13:47.058430+0000 mon.c (mon.2) 169 : audit 0 from='client.? 172.21.15.145:0/2790718999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:47.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:47 smithi189 bash[23614]: audit 2024-03-07T03:13:47.088143+0000 mon.a (mon.0) 1114 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:47.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:47 smithi189 bash[23614]: audit 2024-03-07T03:13:47.103168+0000 mon.a (mon.0) 1115 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:48.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:47 smithi145 bash[72238]: debug 2024-03-07T03:13:47.729+0000 7ff48f3d0200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:13:48.924 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:48 smithi145 bash[30906]: cluster 2024-03-07T03:13:47.410369+0000 mgr.x (mgr.24511) 46 : cluster 0 pgmap v67: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 12 MiB/s wr, 161 op/s 2024-03-07T03:13:48.924 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:48 smithi145 bash[30906]: cluster 2024-03-07T03:13:47.596008+0000 mon.a (mon.0) 1116 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2024-03-07T03:13:48.924 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:48 smithi145 bash[30906]: cluster 2024-03-07T03:13:48.464707+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2024-03-07T03:13:48.924 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:48 smithi145 bash[30906]: audit 2024-03-07T03:13:48.530261+0000 mon.b (mon.1) 76 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:48.924 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:48 smithi145 bash[72238]: debug 2024-03-07T03:13:48.797+0000 7ff48f3d0200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:13:48.925 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:48 smithi145 bash[21339]: cluster 2024-03-07T03:13:47.410369+0000 mgr.x (mgr.24511) 46 : cluster 0 pgmap v67: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 12 MiB/s wr, 161 op/s 2024-03-07T03:13:48.925 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:48 smithi145 bash[21339]: cluster 2024-03-07T03:13:47.596008+0000 mon.a (mon.0) 1116 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2024-03-07T03:13:48.925 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:48 smithi145 bash[21339]: cluster 2024-03-07T03:13:48.464707+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2024-03-07T03:13:48.925 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:48 smithi145 bash[21339]: audit 2024-03-07T03:13:48.530261+0000 mon.b (mon.1) 76 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:48 smithi189 bash[23614]: cluster 2024-03-07T03:13:47.410369+0000 mgr.x (mgr.24511) 46 : cluster 0 pgmap v67: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 12 MiB/s wr, 161 op/s 2024-03-07T03:13:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:48 smithi189 bash[23614]: cluster 2024-03-07T03:13:47.596008+0000 mon.a (mon.0) 1116 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2024-03-07T03:13:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:48 smithi189 bash[23614]: cluster 2024-03-07T03:13:48.464707+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2024-03-07T03:13:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:48 smithi189 bash[23614]: audit 2024-03-07T03:13:48.530261+0000 mon.b (mon.1) 76 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:49.280 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:48 smithi145 bash[72238]: debug 2024-03-07T03:13:48.933+0000 7ff48f3d0200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:13:49.521 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: after_prepare_snap 2024-03-07T03:13:49.558 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:49 smithi145 bash[72238]: debug 2024-03-07T03:13:49.289+0000 7ff48f3d0200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:13:49.558 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:49 smithi145 bash[72238]: debug 2024-03-07T03:13:49.429+0000 7ff48f3d0200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:13:49.910 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:49 smithi145 bash[21339]: audit 2024-03-07T03:13:49.524660+0000 mon.c (mon.2) 170 : audit 0 from='client.? 172.21.15.145:0/35161038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:49.910 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:49 smithi145 bash[30906]: audit 2024-03-07T03:13:49.524660+0000 mon.c (mon.2) 170 : audit 0 from='client.? 172.21.15.145:0/35161038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:50.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:49 smithi189 bash[23614]: audit 2024-03-07T03:13:49.524660+0000 mon.c (mon.2) 170 : audit 0 from='client.? 172.21.15.145:0/35161038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:50.162 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:49 smithi145 bash[72238]: debug 2024-03-07T03:13:49.921+0000 7ff48f3d0200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:13:50.162 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[72238]: debug 2024-03-07T03:13:50.045+0000 7ff48f3d0200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:13:50.162 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[72238]: [07/Mar/2024:03:13:50] ENGINE Bus STARTING 2024-03-07T03:13:50.558 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[72238]: CherryPy Checker: 2024-03-07T03:13:50.559 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[72238]: The Application mounted at '' has an empty config. 2024-03-07T03:13:50.559 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[72238]: [07/Mar/2024:03:13:50] ENGINE Serving on http://:::9283 2024-03-07T03:13:50.559 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[72238]: [07/Mar/2024:03:13:50] ENGINE Bus STARTED 2024-03-07T03:13:50.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:50 smithi189 bash[52103]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:13:50] "GET /metrics HTTP/1.1" 200 38629 "" "Prometheus/2.43.0" 2024-03-07T03:13:50.635 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 41943040 2024-03-07T03:13:50.712 INFO:tasks.workunit.client.0.smithi145.stdout:write: * 0~1000 2024-03-07T03:13:50.731 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:13:50.995 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:13:51.058 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[57902]: ts=2024-03-07T03:13:50.674Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=6 2024-03-07T03:13:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[30906]: cluster 2024-03-07T03:13:49.411091+0000 mgr.x (mgr.24511) 47 : cluster 0 pgmap v70: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:13:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[30906]: cluster 2024-03-07T03:13:49.608632+0000 mon.a (mon.0) 1118 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2024-03-07T03:13:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[30906]: cluster 2024-03-07T03:13:50.067492+0000 mon.a (mon.0) 1119 : cluster 0 Standby manager daemon y restarted 2024-03-07T03:13:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[30906]: cluster 2024-03-07T03:13:50.067678+0000 mon.a (mon.0) 1120 : cluster 0 Standby manager daemon y started 2024-03-07T03:13:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[30906]: audit 2024-03-07T03:13:50.069876+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.? 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-07T03:13:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[30906]: audit 2024-03-07T03:13:50.070940+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.? 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:13:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[30906]: audit 2024-03-07T03:13:50.072901+0000 mon.a (mon.0) 1123 : audit 0 from='mgr.? 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-07T03:13:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[30906]: audit 2024-03-07T03:13:50.074207+0000 mon.a (mon.0) 1124 : audit 0 from='mgr.? 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:13:51.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[21339]: cluster 2024-03-07T03:13:49.411091+0000 mgr.x (mgr.24511) 47 : cluster 0 pgmap v70: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:13:51.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[21339]: cluster 2024-03-07T03:13:49.608632+0000 mon.a (mon.0) 1118 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2024-03-07T03:13:51.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[21339]: cluster 2024-03-07T03:13:50.067492+0000 mon.a (mon.0) 1119 : cluster 0 Standby manager daemon y restarted 2024-03-07T03:13:51.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[21339]: cluster 2024-03-07T03:13:50.067678+0000 mon.a (mon.0) 1120 : cluster 0 Standby manager daemon y started 2024-03-07T03:13:51.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[21339]: audit 2024-03-07T03:13:50.069876+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.? 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-07T03:13:51.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[21339]: audit 2024-03-07T03:13:50.070940+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.? 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:13:51.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[21339]: audit 2024-03-07T03:13:50.072901+0000 mon.a (mon.0) 1123 : audit 0 from='mgr.? 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-07T03:13:51.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:50 smithi145 bash[21339]: audit 2024-03-07T03:13:50.074207+0000 mon.a (mon.0) 1124 : audit 0 from='mgr.? 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:13:51.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:50 smithi189 bash[23614]: cluster 2024-03-07T03:13:49.411091+0000 mgr.x (mgr.24511) 47 : cluster 0 pgmap v70: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:13:51.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:50 smithi189 bash[23614]: cluster 2024-03-07T03:13:49.608632+0000 mon.a (mon.0) 1118 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2024-03-07T03:13:51.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:50 smithi189 bash[23614]: cluster 2024-03-07T03:13:50.067492+0000 mon.a (mon.0) 1119 : cluster 0 Standby manager daemon y restarted 2024-03-07T03:13:51.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:50 smithi189 bash[23614]: cluster 2024-03-07T03:13:50.067678+0000 mon.a (mon.0) 1120 : cluster 0 Standby manager daemon y started 2024-03-07T03:13:51.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:50 smithi189 bash[23614]: audit 2024-03-07T03:13:50.069876+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.? 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-07T03:13:51.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:50 smithi189 bash[23614]: audit 2024-03-07T03:13:50.070940+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.? 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:13:51.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:50 smithi189 bash[23614]: audit 2024-03-07T03:13:50.072901+0000 mon.a (mon.0) 1123 : audit 0 from='mgr.? 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-07T03:13:51.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:50 smithi189 bash[23614]: audit 2024-03-07T03:13:50.074207+0000 mon.a (mon.0) 1124 : audit 0 from='mgr.? 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:13:52.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:51 smithi145 bash[21339]: cluster 2024-03-07T03:13:50.611783+0000 mon.a (mon.0) 1125 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2024-03-07T03:13:52.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:51 smithi145 bash[21339]: cluster 2024-03-07T03:13:50.633585+0000 mon.a (mon.0) 1126 : cluster 0 mgrmap e26: x(active, since 47s), standbys: y 2024-03-07T03:13:52.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:51 smithi145 bash[30906]: cluster 2024-03-07T03:13:50.611783+0000 mon.a (mon.0) 1125 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2024-03-07T03:13:52.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:51 smithi145 bash[30906]: cluster 2024-03-07T03:13:50.633585+0000 mon.a (mon.0) 1126 : cluster 0 mgrmap e26: x(active, since 47s), standbys: y 2024-03-07T03:13:52.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:51 smithi189 bash[23614]: cluster 2024-03-07T03:13:50.611783+0000 mon.a (mon.0) 1125 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2024-03-07T03:13:52.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:51 smithi189 bash[23614]: cluster 2024-03-07T03:13:50.633585+0000 mon.a (mon.0) 1126 : cluster 0 mgrmap e26: x(active, since 47s), standbys: y 2024-03-07T03:13:53.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:52 smithi145 bash[21339]: cluster 2024-03-07T03:13:51.411900+0000 mgr.x (mgr.24511) 48 : cluster 0 pgmap v73: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 44 MiB/s rd, 12 KiB/s wr, 317 op/s 2024-03-07T03:13:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:52 smithi145 bash[21339]: cluster 2024-03-07T03:13:51.640518+0000 mon.a (mon.0) 1127 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2024-03-07T03:13:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:52 smithi145 bash[21339]: audit 2024-03-07T03:13:51.972135+0000 mon.a (mon.0) 1128 : audit 0 from='client.? 172.21.15.145:0/1585035367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:52 smithi145 bash[30906]: cluster 2024-03-07T03:13:51.411900+0000 mgr.x (mgr.24511) 48 : cluster 0 pgmap v73: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 44 MiB/s rd, 12 KiB/s wr, 317 op/s 2024-03-07T03:13:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:52 smithi145 bash[30906]: cluster 2024-03-07T03:13:51.640518+0000 mon.a (mon.0) 1127 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2024-03-07T03:13:53.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:52 smithi145 bash[30906]: audit 2024-03-07T03:13:51.972135+0000 mon.a (mon.0) 1128 : audit 0 from='client.? 172.21.15.145:0/1585035367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:53.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:52 smithi189 bash[23614]: cluster 2024-03-07T03:13:51.411900+0000 mgr.x (mgr.24511) 48 : cluster 0 pgmap v73: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 44 MiB/s rd, 12 KiB/s wr, 317 op/s 2024-03-07T03:13:53.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:52 smithi189 bash[23614]: cluster 2024-03-07T03:13:51.640518+0000 mon.a (mon.0) 1127 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2024-03-07T03:13:53.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:52 smithi189 bash[23614]: audit 2024-03-07T03:13:51.972135+0000 mon.a (mon.0) 1128 : audit 0 from='client.? 172.21.15.145:0/1585035367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:53.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:53 smithi145 bash[21339]: cluster 2024-03-07T03:13:52.653000+0000 mon.a (mon.0) 1129 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2024-03-07T03:13:53.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:53 smithi145 bash[30906]: cluster 2024-03-07T03:13:52.653000+0000 mon.a (mon.0) 1129 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2024-03-07T03:13:53.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:53 smithi189 bash[23614]: cluster 2024-03-07T03:13:52.653000+0000 mon.a (mon.0) 1129 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2024-03-07T03:13:53.831 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:13:53 smithi189 bash[58488]: ts=2024-03-07T03:13:53.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:13:54.215 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.SnapsSnapAfterPrepare (14534 ms) 2024-03-07T03:13:54.215 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.SnapsSnapExpandAfterPrepare 2024-03-07T03:13:54.289 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:13:54.315 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:13:54.422 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:13:54.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:54 smithi145 bash[21339]: cluster 2024-03-07T03:13:53.412626+0000 mgr.x (mgr.24511) 49 : cluster 0 pgmap v76: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 44 MiB/s rd, 12 KiB/s wr, 317 op/s 2024-03-07T03:13:54.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:54 smithi145 bash[21339]: cluster 2024-03-07T03:13:53.661853+0000 mon.a (mon.0) 1130 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2024-03-07T03:13:54.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:54 smithi145 bash[21339]: audit 2024-03-07T03:13:54.483956+0000 mon.a (mon.0) 1131 : audit 0 from='client.? 172.21.15.145:0/2597268511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:54 smithi145 bash[30906]: cluster 2024-03-07T03:13:53.412626+0000 mgr.x (mgr.24511) 49 : cluster 0 pgmap v76: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 44 MiB/s rd, 12 KiB/s wr, 317 op/s 2024-03-07T03:13:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:54 smithi145 bash[30906]: cluster 2024-03-07T03:13:53.661853+0000 mon.a (mon.0) 1130 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2024-03-07T03:13:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:54 smithi145 bash[30906]: audit 2024-03-07T03:13:54.483956+0000 mon.a (mon.0) 1131 : audit 0 from='client.? 172.21.15.145:0/2597268511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:55.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:54 smithi189 bash[23614]: cluster 2024-03-07T03:13:53.412626+0000 mgr.x (mgr.24511) 49 : cluster 0 pgmap v76: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 44 MiB/s rd, 12 KiB/s wr, 317 op/s 2024-03-07T03:13:55.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:54 smithi189 bash[23614]: cluster 2024-03-07T03:13:53.661853+0000 mon.a (mon.0) 1130 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2024-03-07T03:13:55.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:54 smithi189 bash[23614]: audit 2024-03-07T03:13:54.483956+0000 mon.a (mon.0) 1131 : audit 0 from='client.? 172.21.15.145:0/2597268511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:55.684 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 8388609~8388609 2024-03-07T03:13:55.698 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 0~1000 2024-03-07T03:13:55.698 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 1010~1000 2024-03-07T03:13:55.698 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:13:56.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:55 smithi145 bash[21339]: cluster 2024-03-07T03:13:54.673291+0000 mon.a (mon.0) 1132 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2024-03-07T03:13:56.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:55 smithi145 bash[30906]: cluster 2024-03-07T03:13:54.673291+0000 mon.a (mon.0) 1132 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2024-03-07T03:13:56.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:55 smithi189 bash[23614]: cluster 2024-03-07T03:13:54.673291+0000 mon.a (mon.0) 1132 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2024-03-07T03:13:56.795 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:56 smithi145 bash[30906]: cluster 2024-03-07T03:13:55.413430+0000 mgr.x (mgr.24511) 50 : cluster 0 pgmap v79: 137 pgs: 137 active+clean; 689 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 432 op/s 2024-03-07T03:13:56.795 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:56 smithi145 bash[30906]: cluster 2024-03-07T03:13:55.678827+0000 mon.a (mon.0) 1133 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2024-03-07T03:13:56.795 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:56 smithi145 bash[21339]: cluster 2024-03-07T03:13:55.413430+0000 mgr.x (mgr.24511) 50 : cluster 0 pgmap v79: 137 pgs: 137 active+clean; 689 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 432 op/s 2024-03-07T03:13:56.795 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:56 smithi145 bash[21339]: cluster 2024-03-07T03:13:55.678827+0000 mon.a (mon.0) 1133 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2024-03-07T03:13:57.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:56 smithi189 bash[23614]: cluster 2024-03-07T03:13:55.413430+0000 mgr.x (mgr.24511) 50 : cluster 0 pgmap v79: 137 pgs: 137 active+clean; 689 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 432 op/s 2024-03-07T03:13:57.119 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:56 smithi189 bash[23614]: cluster 2024-03-07T03:13:55.678827+0000 mon.a (mon.0) 1133 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2024-03-07T03:13:57.657 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 8388609~1000 2024-03-07T03:13:57.665 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 16777228~1000 2024-03-07T03:13:57.665 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 167772160 2024-03-07T03:13:57.680 INFO:tasks.workunit.client.0.smithi145.stdout:write: 2 83886079~8388609 2024-03-07T03:13:57.702 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:13:57.911 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: [07/Mar/2024:03:13:57] ENGINE Bus STOPPING 2024-03-07T03:13:57.911 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: [07/Mar/2024:03:13:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: [07/Mar/2024:03:13:57] ENGINE Bus STOPPED 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: [07/Mar/2024:03:13:57] ENGINE Bus STARTING 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: debug 2024-03-07T03:13:57.865+0000 7ff3ed044700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: debug 2024-03-07T03:13:57.865+0000 7ff3ed044700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: debug 2024-03-07T03:13:57.865+0000 7ff3ed044700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: debug 2024-03-07T03:13:57.865+0000 7ff3ed044700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: debug 2024-03-07T03:13:57.865+0000 7ff3ed044700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: debug 2024-03-07T03:13:57.869+0000 7ff3f004a700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: debug 2024-03-07T03:13:57.869+0000 7ff3f004a700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: debug 2024-03-07T03:13:57.869+0000 7ff3f004a700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: debug 2024-03-07T03:13:57.869+0000 7ff3f004a700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: debug 2024-03-07T03:13:57.869+0000 7ff3f004a700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: [07/Mar/2024:03:13:57] ENGINE Serving on http://:::9283 2024-03-07T03:13:57.912 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[72238]: [07/Mar/2024:03:13:57] ENGINE Bus STARTED 2024-03-07T03:13:57.913 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: cluster 2024-03-07T03:13:56.689010+0000 mon.a (mon.0) 1134 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2024-03-07T03:13:57.913 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:56.933656+0000 mon.a (mon.0) 1135 : audit 0 from='client.? 172.21.15.145:0/1536604752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:57.913 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.514097+0000 mon.a (mon.0) 1136 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:57.913 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.529770+0000 mon.a (mon.0) 1137 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:57.913 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.531818+0000 mon.b (mon.1) 77 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:57.913 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.533707+0000 mon.b (mon.1) 78 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:57.913 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.547356+0000 mon.a (mon.0) 1138 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:57.913 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.632173+0000 mon.b (mon.1) 79 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:57.913 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.639400+0000 mon.b (mon.1) 80 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-07T03:13:57.913 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.642407+0000 mon.a (mon.0) 1139 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-07T03:13:57.913 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: cluster 2024-03-07T03:13:57.658355+0000 mon.a (mon.0) 1140 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2024-03-07T03:13:57.914 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.659125+0000 mon.a (mon.0) 1141 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-07T03:13:57.914 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: cluster 2024-03-07T03:13:57.659269+0000 mon.a (mon.0) 1142 : cluster 0 mgrmap e27: y(active, starting, since 0.0159603s) 2024-03-07T03:13:57.914 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.660873+0000 mon.a (mon.0) 1143 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:13:57.914 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.661040+0000 mon.a (mon.0) 1144 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:13:57.914 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.661268+0000 mon.a (mon.0) 1145 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:13:57.914 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.663097+0000 mon.a (mon.0) 1146 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:13:57.914 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.663519+0000 mon.a (mon.0) 1147 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.zkinbe"}]: dispatch 2024-03-07T03:13:57.914 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.665297+0000 mon.a (mon.0) 1148 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:13:57.914 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.668238+0000 mon.a (mon.0) 1149 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:13:57.914 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: cluster 2024-03-07T03:13:56.689010+0000 mon.a (mon.0) 1134 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2024-03-07T03:13:57.914 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:56.933656+0000 mon.a (mon.0) 1135 : audit 0 from='client.? 172.21.15.145:0/1536604752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:57.914 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.514097+0000 mon.a (mon.0) 1136 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.529770+0000 mon.a (mon.0) 1137 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.531818+0000 mon.b (mon.1) 77 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.533707+0000 mon.b (mon.1) 78 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.547356+0000 mon.a (mon.0) 1138 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.632173+0000 mon.b (mon.1) 79 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.639400+0000 mon.b (mon.1) 80 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.642407+0000 mon.a (mon.0) 1139 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: cluster 2024-03-07T03:13:57.658355+0000 mon.a (mon.0) 1140 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.659125+0000 mon.a (mon.0) 1141 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: cluster 2024-03-07T03:13:57.659269+0000 mon.a (mon.0) 1142 : cluster 0 mgrmap e27: y(active, starting, since 0.0159603s) 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.660873+0000 mon.a (mon.0) 1143 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.661040+0000 mon.a (mon.0) 1144 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.661268+0000 mon.a (mon.0) 1145 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.663097+0000 mon.a (mon.0) 1146 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.663519+0000 mon.a (mon.0) 1147 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.zkinbe"}]: dispatch 2024-03-07T03:13:57.915 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.665297+0000 mon.a (mon.0) 1148 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:13:57.916 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.668238+0000 mon.a (mon.0) 1149 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:13:57.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.668609+0000 mon.a (mon.0) 1150 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:13:57.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.669076+0000 mon.a (mon.0) 1151 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:13:57.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.669466+0000 mon.a (mon.0) 1152 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:13:57.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.669832+0000 mon.a (mon.0) 1153 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:13:57.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.670191+0000 mon.a (mon.0) 1154 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:13:57.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.670506+0000 mon.a (mon.0) 1155 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:13:57.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.670857+0000 mon.a (mon.0) 1156 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:13:57.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.671234+0000 mon.a (mon.0) 1157 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:13:57.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.671629+0000 mon.a (mon.0) 1158 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:13:57.916 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[21339]: audit 2024-03-07T03:13:57.672255+0000 mon.a (mon.0) 1159 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:13:57.917 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.668609+0000 mon.a (mon.0) 1150 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:13:57.917 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.669076+0000 mon.a (mon.0) 1151 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:13:57.917 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.669466+0000 mon.a (mon.0) 1152 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:13:57.917 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.669832+0000 mon.a (mon.0) 1153 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:13:57.917 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.670191+0000 mon.a (mon.0) 1154 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:13:57.917 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.670506+0000 mon.a (mon.0) 1155 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:13:57.917 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.670857+0000 mon.a (mon.0) 1156 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:13:57.917 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.671234+0000 mon.a (mon.0) 1157 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:13:57.917 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.671629+0000 mon.a (mon.0) 1158 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:13:57.917 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:57 smithi145 bash[30906]: audit 2024-03-07T03:13:57.672255+0000 mon.a (mon.0) 1159 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:13:57.918 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[52103]: debug 2024-03-07T03:13:57.650+0000 7f9757827700 -1 mgr handle_mgr_map I was active but no longer am 2024-03-07T03:13:57.918 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[52103]: ignoring --setuser ceph since I am not root 2024-03-07T03:13:57.918 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[52103]: ignoring --setgroup ceph since I am not root 2024-03-07T03:13:57.918 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[52103]: debug 2024-03-07T03:13:57.722+0000 7f770d65a700 1 -- 172.21.15.189:0/591870823 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c1298d25a0 con 0x55c12a652400 2024-03-07T03:13:57.918 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[52103]: debug 2024-03-07T03:13:57.878+0000 7f771743e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:13:57.918 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: cluster 2024-03-07T03:13:56.689010+0000 mon.a (mon.0) 1134 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2024-03-07T03:13:57.918 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:56.933656+0000 mon.a (mon.0) 1135 : audit 0 from='client.? 172.21.15.145:0/1536604752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:57.918 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.514097+0000 mon.a (mon.0) 1136 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.529770+0000 mon.a (mon.0) 1137 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.531818+0000 mon.b (mon.1) 77 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.533707+0000 mon.b (mon.1) 78 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.547356+0000 mon.a (mon.0) 1138 : audit 1 from='mgr.24511 ' entity='mgr.x' 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.632173+0000 mon.b (mon.1) 79 : audit 0 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.639400+0000 mon.b (mon.1) 80 : audit 1 from='mgr.24511 172.21.15.189:0/1322080454' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.642407+0000 mon.a (mon.0) 1139 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: cluster 2024-03-07T03:13:57.658355+0000 mon.a (mon.0) 1140 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.659125+0000 mon.a (mon.0) 1141 : audit 1 from='mgr.24511 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: cluster 2024-03-07T03:13:57.659269+0000 mon.a (mon.0) 1142 : cluster 0 mgrmap e27: y(active, starting, since 0.0159603s) 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.660873+0000 mon.a (mon.0) 1143 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.661040+0000 mon.a (mon.0) 1144 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.661268+0000 mon.a (mon.0) 1145 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.663097+0000 mon.a (mon.0) 1146 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:13:57.919 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.663519+0000 mon.a (mon.0) 1147 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.zkinbe"}]: dispatch 2024-03-07T03:13:57.920 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.665297+0000 mon.a (mon.0) 1148 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:13:57.920 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.668238+0000 mon.a (mon.0) 1149 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:13:57.920 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.668609+0000 mon.a (mon.0) 1150 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:13:57.920 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.669076+0000 mon.a (mon.0) 1151 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:13:57.920 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.669466+0000 mon.a (mon.0) 1152 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:13:57.920 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.669832+0000 mon.a (mon.0) 1153 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:13:57.920 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.670191+0000 mon.a (mon.0) 1154 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:13:57.920 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.670506+0000 mon.a (mon.0) 1155 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:13:57.920 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.670857+0000 mon.a (mon.0) 1156 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:13:57.920 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.671234+0000 mon.a (mon.0) 1157 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:13:57.921 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.671629+0000 mon.a (mon.0) 1158 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:13:57.921 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:57 smithi189 bash[23614]: audit 2024-03-07T03:13:57.672255+0000 mon.a (mon.0) 1159 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:13:59.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[30906]: cluster 2024-03-07T03:13:57.724988+0000 mon.a (mon.0) 1160 : cluster 1 Manager daemon y is now available 2024-03-07T03:13:59.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[30906]: audit 2024-03-07T03:13:57.776505+0000 mon.a (mon.0) 1161 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:59.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[30906]: audit 2024-03-07T03:13:57.778370+0000 mon.a (mon.0) 1162 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:59.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[30906]: audit 2024-03-07T03:13:57.804727+0000 mon.a (mon.0) 1163 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:59.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[30906]: audit 2024-03-07T03:13:57.815602+0000 mon.a (mon.0) 1164 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:13:59.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[30906]: audit 2024-03-07T03:13:57.845071+0000 mon.a (mon.0) 1165 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:13:59.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[30906]: cephadm 2024-03-07T03:13:58.216585+0000 mgr.y (mgr.15132) 1 : cephadm 1 [07/Mar/2024:03:13:58] ENGINE Bus STARTING 2024-03-07T03:13:59.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[30906]: cephadm 2024-03-07T03:13:58.318615+0000 mgr.y (mgr.15132) 2 : cephadm 1 [07/Mar/2024:03:13:58] ENGINE Serving on http://172.21.15.145:8765 2024-03-07T03:13:59.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[30906]: cluster 2024-03-07T03:13:58.468544+0000 mon.a (mon.0) 1166 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2024-03-07T03:13:59.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[21339]: cluster 2024-03-07T03:13:57.724988+0000 mon.a (mon.0) 1160 : cluster 1 Manager daemon y is now available 2024-03-07T03:13:59.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[21339]: audit 2024-03-07T03:13:57.776505+0000 mon.a (mon.0) 1161 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:59.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[21339]: audit 2024-03-07T03:13:57.778370+0000 mon.a (mon.0) 1162 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:59.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[21339]: audit 2024-03-07T03:13:57.804727+0000 mon.a (mon.0) 1163 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:59.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[21339]: audit 2024-03-07T03:13:57.815602+0000 mon.a (mon.0) 1164 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:13:59.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[21339]: audit 2024-03-07T03:13:57.845071+0000 mon.a (mon.0) 1165 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:13:59.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[21339]: cephadm 2024-03-07T03:13:58.216585+0000 mgr.y (mgr.15132) 1 : cephadm 1 [07/Mar/2024:03:13:58] ENGINE Bus STARTING 2024-03-07T03:13:59.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[21339]: cephadm 2024-03-07T03:13:58.318615+0000 mgr.y (mgr.15132) 2 : cephadm 1 [07/Mar/2024:03:13:58] ENGINE Serving on http://172.21.15.145:8765 2024-03-07T03:13:59.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:58 smithi145 bash[21339]: cluster 2024-03-07T03:13:58.468544+0000 mon.a (mon.0) 1166 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2024-03-07T03:13:59.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:58 smithi189 bash[23614]: cluster 2024-03-07T03:13:57.724988+0000 mon.a (mon.0) 1160 : cluster 1 Manager daemon y is now available 2024-03-07T03:13:59.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:58 smithi189 bash[23614]: audit 2024-03-07T03:13:57.776505+0000 mon.a (mon.0) 1161 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:59.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:58 smithi189 bash[23614]: audit 2024-03-07T03:13:57.778370+0000 mon.a (mon.0) 1162 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:59.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:58 smithi189 bash[23614]: audit 2024-03-07T03:13:57.804727+0000 mon.a (mon.0) 1163 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:59.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:58 smithi189 bash[23614]: audit 2024-03-07T03:13:57.815602+0000 mon.a (mon.0) 1164 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:13:59.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:58 smithi189 bash[23614]: audit 2024-03-07T03:13:57.845071+0000 mon.a (mon.0) 1165 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:13:59.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:58 smithi189 bash[23614]: cephadm 2024-03-07T03:13:58.216585+0000 mgr.y (mgr.15132) 1 : cephadm 1 [07/Mar/2024:03:13:58] ENGINE Bus STARTING 2024-03-07T03:13:59.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:58 smithi189 bash[23614]: cephadm 2024-03-07T03:13:58.318615+0000 mgr.y (mgr.15132) 2 : cephadm 1 [07/Mar/2024:03:13:58] ENGINE Serving on http://172.21.15.145:8765 2024-03-07T03:13:59.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:58 smithi189 bash[23614]: cluster 2024-03-07T03:13:58.468544+0000 mon.a (mon.0) 1166 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2024-03-07T03:13:59.493 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:13:59.550 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 83886079~1 2024-03-07T03:13:59.574 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:13:59.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:59 smithi189 bash[52103]: debug 2024-03-07T03:13:59.510+0000 7f771743e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:13:59.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:59 smithi189 bash[52103]: debug 2024-03-07T03:13:59.618+0000 7f771743e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:13:59.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:59 smithi189 bash[23614]: cephadm 2024-03-07T03:13:58.436872+0000 mgr.y (mgr.15132) 3 : cephadm 1 [07/Mar/2024:03:13:58] ENGINE Serving on https://172.21.15.145:7150 2024-03-07T03:13:59.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:59 smithi189 bash[23614]: cephadm 2024-03-07T03:13:58.437070+0000 mgr.y (mgr.15132) 4 : cephadm 1 [07/Mar/2024:03:13:58] ENGINE Bus STARTED 2024-03-07T03:13:59.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:59 smithi189 bash[23614]: cluster 2024-03-07T03:13:58.714023+0000 mon.a (mon.0) 1167 : cluster 0 mgrmap e28: y(active, since 1.07071s) 2024-03-07T03:13:59.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:59 smithi189 bash[23614]: cluster 2024-03-07T03:13:58.755614+0000 mgr.y (mgr.15132) 5 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:13:59.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:59 smithi189 bash[23614]: audit 2024-03-07T03:13:59.372336+0000 mon.c (mon.2) 171 : audit 0 from='client.? 172.21.15.145:0/3761484387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:13:59.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:13:59 smithi189 bash[23614]: cluster 2024-03-07T03:13:59.479621+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2024-03-07T03:14:00.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:59 smithi145 bash[21339]: cephadm 2024-03-07T03:13:58.436872+0000 mgr.y (mgr.15132) 3 : cephadm 1 [07/Mar/2024:03:13:58] ENGINE Serving on https://172.21.15.145:7150 2024-03-07T03:14:00.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:59 smithi145 bash[21339]: cephadm 2024-03-07T03:13:58.437070+0000 mgr.y (mgr.15132) 4 : cephadm 1 [07/Mar/2024:03:13:58] ENGINE Bus STARTED 2024-03-07T03:14:00.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:59 smithi145 bash[21339]: cluster 2024-03-07T03:13:58.714023+0000 mon.a (mon.0) 1167 : cluster 0 mgrmap e28: y(active, since 1.07071s) 2024-03-07T03:14:00.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:59 smithi145 bash[21339]: cluster 2024-03-07T03:13:58.755614+0000 mgr.y (mgr.15132) 5 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:00.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:59 smithi145 bash[21339]: audit 2024-03-07T03:13:59.372336+0000 mon.c (mon.2) 171 : audit 0 from='client.? 172.21.15.145:0/3761484387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:00.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:13:59 smithi145 bash[21339]: cluster 2024-03-07T03:13:59.479621+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2024-03-07T03:14:00.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:59 smithi145 bash[30906]: cephadm 2024-03-07T03:13:58.436872+0000 mgr.y (mgr.15132) 3 : cephadm 1 [07/Mar/2024:03:13:58] ENGINE Serving on https://172.21.15.145:7150 2024-03-07T03:14:00.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:59 smithi145 bash[30906]: cephadm 2024-03-07T03:13:58.437070+0000 mgr.y (mgr.15132) 4 : cephadm 1 [07/Mar/2024:03:13:58] ENGINE Bus STARTED 2024-03-07T03:14:00.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:59 smithi145 bash[30906]: cluster 2024-03-07T03:13:58.714023+0000 mon.a (mon.0) 1167 : cluster 0 mgrmap e28: y(active, since 1.07071s) 2024-03-07T03:14:00.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:59 smithi145 bash[30906]: cluster 2024-03-07T03:13:58.755614+0000 mgr.y (mgr.15132) 5 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:00.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:59 smithi145 bash[30906]: audit 2024-03-07T03:13:59.372336+0000 mon.c (mon.2) 171 : audit 0 from='client.? 172.21.15.145:0/3761484387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:00.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:13:59 smithi145 bash[30906]: cluster 2024-03-07T03:13:59.479621+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2024-03-07T03:14:00.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:13:59 smithi189 bash[52103]: debug 2024-03-07T03:13:59.886+0000 7f771743e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:14:01.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:00 smithi145 bash[21339]: cluster 2024-03-07T03:13:59.669348+0000 mgr.y (mgr.15132) 6 : cluster 0 pgmap v6: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:01.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:00 smithi145 bash[21339]: cluster 2024-03-07T03:14:00.485321+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e227: 8 total, 8 up, 8 in 2024-03-07T03:14:01.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:00 smithi145 bash[30906]: cluster 2024-03-07T03:13:59.669348+0000 mgr.y (mgr.15132) 6 : cluster 0 pgmap v6: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:01.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:00 smithi145 bash[30906]: cluster 2024-03-07T03:14:00.485321+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e227: 8 total, 8 up, 8 in 2024-03-07T03:14:01.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:00 smithi189 bash[23614]: cluster 2024-03-07T03:13:59.669348+0000 mgr.y (mgr.15132) 6 : cluster 0 pgmap v6: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:01.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:00 smithi189 bash[23614]: cluster 2024-03-07T03:14:00.485321+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e227: 8 total, 8 up, 8 in 2024-03-07T03:14:01.081 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:00 smithi189 bash[52103]: debug 2024-03-07T03:14:00.698+0000 7f771743e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:14:01.081 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:00 smithi189 bash[52103]: debug 2024-03-07T03:14:00.814+0000 7f771743e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:14:01.081 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:00 smithi189 bash[52103]: debug 2024-03-07T03:14:00.918+0000 7f771743e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:14:01.580 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:01 smithi189 bash[52103]: debug 2024-03-07T03:14:01.278+0000 7f771743e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:14:01.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:01 smithi189 bash[52103]: debug 2024-03-07T03:14:01.390+0000 7f771743e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:14:01.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:01 smithi189 bash[52103]: debug 2024-03-07T03:14:01.506+0000 7f771743e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:14:02.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:01 smithi145 bash[21339]: cluster 2024-03-07T03:14:00.727364+0000 mon.a (mon.0) 1170 : cluster 0 mgrmap e29: y(active, since 3s) 2024-03-07T03:14:02.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:01 smithi145 bash[21339]: cluster 2024-03-07T03:14:01.486736+0000 mon.a (mon.0) 1171 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2024-03-07T03:14:02.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:01 smithi145 bash[30906]: cluster 2024-03-07T03:14:00.727364+0000 mon.a (mon.0) 1170 : cluster 0 mgrmap e29: y(active, since 3s) 2024-03-07T03:14:02.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:01 smithi145 bash[30906]: cluster 2024-03-07T03:14:01.486736+0000 mon.a (mon.0) 1171 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2024-03-07T03:14:02.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:01 smithi189 bash[23614]: cluster 2024-03-07T03:14:00.727364+0000 mon.a (mon.0) 1170 : cluster 0 mgrmap e29: y(active, since 3s) 2024-03-07T03:14:02.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:01 smithi189 bash[23614]: cluster 2024-03-07T03:14:01.486736+0000 mon.a (mon.0) 1171 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2024-03-07T03:14:02.081 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:01 smithi189 bash[52103]: debug 2024-03-07T03:14:01.866+0000 7f771743e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:14:02.369 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:02 smithi189 bash[52103]: debug 2024-03-07T03:14:02.110+0000 7f771743e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:14:02.719 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:02 smithi189 bash[52103]: debug 2024-03-07T03:14:02.374+0000 7f771743e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:14:02.719 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:02 smithi189 bash[52103]: debug 2024-03-07T03:14:02.498+0000 7f771743e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:14:03.048 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:02 smithi189 bash[23614]: cluster 2024-03-07T03:14:01.670109+0000 mgr.y (mgr.15132) 7 : cluster 0 pgmap v9: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:03.048 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:02 smithi189 bash[23614]: audit 2024-03-07T03:14:01.816339+0000 mon.a (mon.0) 1172 : audit 0 from='client.? 172.21.15.145:0/3483161339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:03.048 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:02 smithi189 bash[23614]: cluster 2024-03-07T03:14:02.488665+0000 mon.a (mon.0) 1173 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2024-03-07T03:14:03.048 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:02 smithi189 bash[52103]: debug 2024-03-07T03:14:02.794+0000 7f771743e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:14:03.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:02 smithi145 bash[21339]: cluster 2024-03-07T03:14:01.670109+0000 mgr.y (mgr.15132) 7 : cluster 0 pgmap v9: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:03.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:02 smithi145 bash[21339]: audit 2024-03-07T03:14:01.816339+0000 mon.a (mon.0) 1172 : audit 0 from='client.? 172.21.15.145:0/3483161339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:03.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:02 smithi145 bash[21339]: cluster 2024-03-07T03:14:02.488665+0000 mon.a (mon.0) 1173 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2024-03-07T03:14:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:02 smithi145 bash[30906]: cluster 2024-03-07T03:14:01.670109+0000 mgr.y (mgr.15132) 7 : cluster 0 pgmap v9: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:02 smithi145 bash[30906]: audit 2024-03-07T03:14:01.816339+0000 mon.a (mon.0) 1172 : audit 0 from='client.? 172.21.15.145:0/3483161339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:02 smithi145 bash[30906]: cluster 2024-03-07T03:14:02.488665+0000 mon.a (mon.0) 1173 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2024-03-07T03:14:03.257 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: after_prepare_snap 2024-03-07T03:14:03.330 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:03 smithi189 bash[52103]: debug 2024-03-07T03:14:03.054+0000 7f771743e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:14:03.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:03 smithi189 bash[52103]: debug 2024-03-07T03:14:03.490+0000 7f771743e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:14:03.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:03 smithi189 bash[52103]: debug 2024-03-07T03:14:03.618+0000 7f771743e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:14:03.831 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:03 smithi189 bash[58488]: ts=2024-03-07T03:14:03.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:14:04.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:03 smithi189 bash[52103]: debug 2024-03-07T03:14:03.902+0000 7f771743e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:14:04.751 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 167772160 2024-03-07T03:14:04.783 INFO:tasks.workunit.client.0.smithi145.stdout:write: * 83886080~1000 2024-03-07T03:14:04.798 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:14:04.999 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:14:05.009 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:04 smithi189 bash[23614]: cluster 2024-03-07T03:14:03.670545+0000 mgr.y (mgr.15132) 8 : cluster 0 pgmap v11: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:05.009 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:04 smithi189 bash[23614]: cluster 2024-03-07T03:14:03.739278+0000 mon.a (mon.0) 1174 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2024-03-07T03:14:05.010 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:04 smithi189 bash[23614]: audit 2024-03-07T03:14:04.005187+0000 mon.a (mon.0) 1175 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:05.010 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:04 smithi189 bash[23614]: audit 2024-03-07T03:14:04.013704+0000 mon.a (mon.0) 1176 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:05.010 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:04 smithi189 bash[23614]: audit 2024-03-07T03:14:04.249805+0000 mon.a (mon.0) 1177 : audit 0 from='client.? 172.21.15.145:0/1606798158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:05.010 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:04 smithi189 bash[23614]: audit 2024-03-07T03:14:04.262551+0000 mon.a (mon.0) 1178 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:05.010 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:04 smithi189 bash[23614]: audit 2024-03-07T03:14:04.268148+0000 mon.a (mon.0) 1179 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:05.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[21339]: cluster 2024-03-07T03:14:03.670545+0000 mgr.y (mgr.15132) 8 : cluster 0 pgmap v11: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:05.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[21339]: cluster 2024-03-07T03:14:03.739278+0000 mon.a (mon.0) 1174 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2024-03-07T03:14:05.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[21339]: audit 2024-03-07T03:14:04.005187+0000 mon.a (mon.0) 1175 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:05.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[21339]: audit 2024-03-07T03:14:04.013704+0000 mon.a (mon.0) 1176 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:05.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[21339]: audit 2024-03-07T03:14:04.249805+0000 mon.a (mon.0) 1177 : audit 0 from='client.? 172.21.15.145:0/1606798158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:05.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[21339]: audit 2024-03-07T03:14:04.262551+0000 mon.a (mon.0) 1178 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:05.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[21339]: audit 2024-03-07T03:14:04.268148+0000 mon.a (mon.0) 1179 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[30906]: cluster 2024-03-07T03:14:03.670545+0000 mgr.y (mgr.15132) 8 : cluster 0 pgmap v11: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[30906]: cluster 2024-03-07T03:14:03.739278+0000 mon.a (mon.0) 1174 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2024-03-07T03:14:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[30906]: audit 2024-03-07T03:14:04.005187+0000 mon.a (mon.0) 1175 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[30906]: audit 2024-03-07T03:14:04.013704+0000 mon.a (mon.0) 1176 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[30906]: audit 2024-03-07T03:14:04.249805+0000 mon.a (mon.0) 1177 : audit 0 from='client.? 172.21.15.145:0/1606798158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[30906]: audit 2024-03-07T03:14:04.262551+0000 mon.a (mon.0) 1178 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:04 smithi145 bash[30906]: audit 2024-03-07T03:14:04.268148+0000 mon.a (mon.0) 1179 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:05.330 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:05 smithi189 bash[52103]: debug 2024-03-07T03:14:05.014+0000 7f771743e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:14:05.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:05 smithi189 bash[52103]: debug 2024-03-07T03:14:05.142+0000 7f771743e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:14:05.748 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:05 smithi189 bash[52103]: debug 2024-03-07T03:14:05.454+0000 7f771743e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:14:05.748 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:05 smithi189 bash[52103]: debug 2024-03-07T03:14:05.570+0000 7f771743e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:14:06.033 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:05 smithi189 bash[23614]: cluster 2024-03-07T03:14:04.738776+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2024-03-07T03:14:06.033 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:05 smithi189 bash[23614]: audit 2024-03-07T03:14:05.176498+0000 mon.a (mon.0) 1181 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:06.033 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:05 smithi189 bash[23614]: audit 2024-03-07T03:14:05.186817+0000 mon.a (mon.0) 1182 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:06.033 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:05 smithi189 bash[23614]: audit 2024-03-07T03:14:05.333853+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:06.033 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:05 smithi189 bash[23614]: audit 2024-03-07T03:14:05.343509+0000 mon.a (mon.0) 1184 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:06.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:05 smithi145 bash[30906]: cluster 2024-03-07T03:14:04.738776+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2024-03-07T03:14:06.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:05 smithi145 bash[30906]: audit 2024-03-07T03:14:05.176498+0000 mon.a (mon.0) 1181 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:06.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:05 smithi145 bash[30906]: audit 2024-03-07T03:14:05.186817+0000 mon.a (mon.0) 1182 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:06.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:05 smithi145 bash[30906]: audit 2024-03-07T03:14:05.333853+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:06.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:05 smithi145 bash[30906]: audit 2024-03-07T03:14:05.343509+0000 mon.a (mon.0) 1184 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:05 smithi145 bash[21339]: cluster 2024-03-07T03:14:04.738776+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2024-03-07T03:14:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:05 smithi145 bash[21339]: audit 2024-03-07T03:14:05.176498+0000 mon.a (mon.0) 1181 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:05 smithi145 bash[21339]: audit 2024-03-07T03:14:05.186817+0000 mon.a (mon.0) 1182 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:05 smithi145 bash[21339]: audit 2024-03-07T03:14:05.333853+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:05 smithi145 bash[21339]: audit 2024-03-07T03:14:05.343509+0000 mon.a (mon.0) 1184 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:06.291 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[52103]: debug 2024-03-07T03:14:06.038+0000 7f771743e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:14:06.291 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[52103]: debug 2024-03-07T03:14:06.166+0000 7f771743e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:14:06.291 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[52103]: [07/Mar/2024:03:14:06] ENGINE Bus STARTING 2024-03-07T03:14:06.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[52103]: CherryPy Checker: 2024-03-07T03:14:06.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[52103]: The Application mounted at '' has an empty config. 2024-03-07T03:14:06.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[52103]: [07/Mar/2024:03:14:06] ENGINE Serving on http://:::9283 2024-03-07T03:14:06.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[52103]: [07/Mar/2024:03:14:06] ENGINE Bus STARTED 2024-03-07T03:14:07.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[21339]: cluster 2024-03-07T03:14:05.671228+0000 mgr.y (mgr.15132) 9 : cluster 0 pgmap v14: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 11 KiB/s wr, 292 op/s 2024-03-07T03:14:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[21339]: cluster 2024-03-07T03:14:05.755452+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e232: 8 total, 8 up, 8 in 2024-03-07T03:14:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[21339]: cluster 2024-03-07T03:14:06.199225+0000 mon.a (mon.0) 1186 : cluster 0 Standby manager daemon x started 2024-03-07T03:14:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[21339]: audit 2024-03-07T03:14:06.199754+0000 mon.b (mon.1) 81 : audit 0 from='mgr.? 172.21.15.189:0/448915378' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:14:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[21339]: audit 2024-03-07T03:14:06.201550+0000 mon.b (mon.1) 82 : audit 0 from='mgr.? 172.21.15.189:0/448915378' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:14:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[21339]: audit 2024-03-07T03:14:06.203171+0000 mon.b (mon.1) 83 : audit 0 from='mgr.? 172.21.15.189:0/448915378' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:14:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[21339]: audit 2024-03-07T03:14:06.203470+0000 mon.b (mon.1) 84 : audit 0 from='mgr.? 172.21.15.189:0/448915378' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:14:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[21339]: audit 2024-03-07T03:14:06.691161+0000 mon.a (mon.0) 1187 : audit 0 from='client.? 172.21.15.145:0/3034112131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:07.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[30906]: cluster 2024-03-07T03:14:05.671228+0000 mgr.y (mgr.15132) 9 : cluster 0 pgmap v14: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 11 KiB/s wr, 292 op/s 2024-03-07T03:14:07.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[30906]: cluster 2024-03-07T03:14:05.755452+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e232: 8 total, 8 up, 8 in 2024-03-07T03:14:07.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[30906]: cluster 2024-03-07T03:14:06.199225+0000 mon.a (mon.0) 1186 : cluster 0 Standby manager daemon x started 2024-03-07T03:14:07.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[30906]: audit 2024-03-07T03:14:06.199754+0000 mon.b (mon.1) 81 : audit 0 from='mgr.? 172.21.15.189:0/448915378' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:14:07.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[30906]: audit 2024-03-07T03:14:06.201550+0000 mon.b (mon.1) 82 : audit 0 from='mgr.? 172.21.15.189:0/448915378' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:14:07.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[30906]: audit 2024-03-07T03:14:06.203171+0000 mon.b (mon.1) 83 : audit 0 from='mgr.? 172.21.15.189:0/448915378' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:14:07.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[30906]: audit 2024-03-07T03:14:06.203470+0000 mon.b (mon.1) 84 : audit 0 from='mgr.? 172.21.15.189:0/448915378' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:14:07.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:06 smithi145 bash[30906]: audit 2024-03-07T03:14:06.691161+0000 mon.a (mon.0) 1187 : audit 0 from='client.? 172.21.15.145:0/3034112131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:07.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[23614]: cluster 2024-03-07T03:14:05.671228+0000 mgr.y (mgr.15132) 9 : cluster 0 pgmap v14: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 11 KiB/s wr, 292 op/s 2024-03-07T03:14:07.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[23614]: cluster 2024-03-07T03:14:05.755452+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e232: 8 total, 8 up, 8 in 2024-03-07T03:14:07.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[23614]: cluster 2024-03-07T03:14:06.199225+0000 mon.a (mon.0) 1186 : cluster 0 Standby manager daemon x started 2024-03-07T03:14:07.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[23614]: audit 2024-03-07T03:14:06.199754+0000 mon.b (mon.1) 81 : audit 0 from='mgr.? 172.21.15.189:0/448915378' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:14:07.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[23614]: audit 2024-03-07T03:14:06.201550+0000 mon.b (mon.1) 82 : audit 0 from='mgr.? 172.21.15.189:0/448915378' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:14:07.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[23614]: audit 2024-03-07T03:14:06.203171+0000 mon.b (mon.1) 83 : audit 0 from='mgr.? 172.21.15.189:0/448915378' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:14:07.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[23614]: audit 2024-03-07T03:14:06.203470+0000 mon.b (mon.1) 84 : audit 0 from='mgr.? 172.21.15.189:0/448915378' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:14:07.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:06 smithi189 bash[23614]: audit 2024-03-07T03:14:06.691161+0000 mon.a (mon.0) 1187 : audit 0 from='client.? 172.21.15.145:0/3034112131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:08.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:07 smithi145 bash[30906]: cluster 2024-03-07T03:14:06.777225+0000 mon.a (mon.0) 1188 : cluster 0 mgrmap e30: y(active, since 9s), standbys: x 2024-03-07T03:14:08.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:07 smithi145 bash[30906]: audit 2024-03-07T03:14:06.778351+0000 mon.a (mon.0) 1189 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:14:08.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:07 smithi145 bash[30906]: cluster 2024-03-07T03:14:06.783728+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e233: 8 total, 8 up, 8 in 2024-03-07T03:14:08.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:07 smithi145 bash[21339]: cluster 2024-03-07T03:14:06.777225+0000 mon.a (mon.0) 1188 : cluster 0 mgrmap e30: y(active, since 9s), standbys: x 2024-03-07T03:14:08.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:07 smithi145 bash[21339]: audit 2024-03-07T03:14:06.778351+0000 mon.a (mon.0) 1189 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:14:08.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:07 smithi145 bash[21339]: cluster 2024-03-07T03:14:06.783728+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e233: 8 total, 8 up, 8 in 2024-03-07T03:14:08.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:07 smithi189 bash[23614]: cluster 2024-03-07T03:14:06.777225+0000 mon.a (mon.0) 1188 : cluster 0 mgrmap e30: y(active, since 9s), standbys: x 2024-03-07T03:14:08.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:07 smithi189 bash[23614]: audit 2024-03-07T03:14:06.778351+0000 mon.a (mon.0) 1189 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:14:08.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:07 smithi189 bash[23614]: cluster 2024-03-07T03:14:06.783728+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e233: 8 total, 8 up, 8 in 2024-03-07T03:14:08.517 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.SnapsSnapExpandAfterPrepare (14303 ms) 2024-03-07T03:14:08.518 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.Clone 2024-03-07T03:14:08.605 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:14:08.634 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 0~8388609 2024-03-07T03:14:08.753 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 16777218~8388609 2024-03-07T03:14:08.821 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap 2024-03-07T03:14:09.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:08 smithi145 bash[21339]: cluster 2024-03-07T03:14:07.672030+0000 mgr.y (mgr.15132) 10 : cluster 0 pgmap v17: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 11 KiB/s wr, 305 op/s 2024-03-07T03:14:09.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:08 smithi145 bash[21339]: cluster 2024-03-07T03:14:07.796686+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2024-03-07T03:14:09.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:08 smithi145 bash[30906]: cluster 2024-03-07T03:14:07.672030+0000 mgr.y (mgr.15132) 10 : cluster 0 pgmap v17: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 11 KiB/s wr, 305 op/s 2024-03-07T03:14:09.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:08 smithi145 bash[30906]: cluster 2024-03-07T03:14:07.796686+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2024-03-07T03:14:09.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:08 smithi189 bash[23614]: cluster 2024-03-07T03:14:07.672030+0000 mgr.y (mgr.15132) 10 : cluster 0 pgmap v17: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 11 KiB/s wr, 305 op/s 2024-03-07T03:14:09.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:08 smithi189 bash[23614]: cluster 2024-03-07T03:14:07.796686+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2024-03-07T03:14:10.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:09 smithi145 bash[21339]: audit 2024-03-07T03:14:09.150469+0000 mon.c (mon.2) 172 : audit 0 from='client.? 172.21.15.145:0/1438592725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:10.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:09 smithi145 bash[30906]: audit 2024-03-07T03:14:09.150469+0000 mon.c (mon.2) 172 : audit 0 from='client.? 172.21.15.145:0/1438592725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:10.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:09 smithi189 bash[23614]: audit 2024-03-07T03:14:09.150469+0000 mon.c (mon.2) 172 : audit 0 from='client.? 172.21.15.145:0/1438592725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:10.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:10 smithi189 bash[52103]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:14:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-07T03:14:10.827 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap 2024-03-07T03:14:10.856 INFO:tasks.workunit.client.0.smithi145.stdout:clone image105 -> image108 2024-03-07T03:14:11.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:10 smithi145 bash[21339]: cluster 2024-03-07T03:14:09.673676+0000 mgr.y (mgr.15132) 11 : cluster 0 pgmap v19: 137 pgs: 137 active+clean; 741 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 7.3 MiB/s wr, 339 op/s 2024-03-07T03:14:11.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:10 smithi145 bash[21339]: cluster 2024-03-07T03:14:09.809079+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2024-03-07T03:14:11.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:10 smithi145 bash[30906]: cluster 2024-03-07T03:14:09.673676+0000 mgr.y (mgr.15132) 11 : cluster 0 pgmap v19: 137 pgs: 137 active+clean; 741 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 7.3 MiB/s wr, 339 op/s 2024-03-07T03:14:11.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:10 smithi145 bash[30906]: cluster 2024-03-07T03:14:09.809079+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2024-03-07T03:14:11.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:10 smithi189 bash[23614]: cluster 2024-03-07T03:14:09.673676+0000 mgr.y (mgr.15132) 11 : cluster 0 pgmap v19: 137 pgs: 137 active+clean; 741 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 7.3 MiB/s wr, 339 op/s 2024-03-07T03:14:11.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:10 smithi189 bash[23614]: cluster 2024-03-07T03:14:09.809079+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2024-03-07T03:14:11.108 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 0~1000 2024-03-07T03:14:11.201 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 1010~1000 2024-03-07T03:14:11.207 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap 2024-03-07T03:14:12.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:11 smithi189 bash[23614]: cluster 2024-03-07T03:14:10.813670+0000 mon.a (mon.0) 1193 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2024-03-07T03:14:12.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:11 smithi189 bash[23614]: audit 2024-03-07T03:14:11.603008+0000 mon.a (mon.0) 1194 : audit 0 from='client.? 172.21.15.145:0/3517362797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:12.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:11 smithi145 bash[21339]: cluster 2024-03-07T03:14:10.813670+0000 mon.a (mon.0) 1193 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2024-03-07T03:14:12.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:11 smithi145 bash[21339]: audit 2024-03-07T03:14:11.603008+0000 mon.a (mon.0) 1194 : audit 0 from='client.? 172.21.15.145:0/3517362797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:12.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:11 smithi145 bash[30906]: cluster 2024-03-07T03:14:10.813670+0000 mon.a (mon.0) 1193 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2024-03-07T03:14:12.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:11 smithi145 bash[30906]: audit 2024-03-07T03:14:11.603008+0000 mon.a (mon.0) 1194 : audit 0 from='client.? 172.21.15.145:0/3517362797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:12.845 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap 2024-03-07T03:14:12.872 INFO:tasks.workunit.client.0.smithi145.stdout:clone image108 -> image110 2024-03-07T03:14:13.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:12 smithi189 bash[23614]: cluster 2024-03-07T03:14:11.674224+0000 mgr.y (mgr.15132) 12 : cluster 0 pgmap v22: 137 pgs: 137 active+clean; 741 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 7.4 MiB/s wr, 342 op/s 2024-03-07T03:14:13.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:12 smithi189 bash[23614]: cluster 2024-03-07T03:14:11.831218+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2024-03-07T03:14:13.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:12 smithi189 bash[23614]: audit 2024-03-07T03:14:12.805714+0000 mon.a (mon.0) 1196 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:13.098 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 1000~1000 2024-03-07T03:14:13.197 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 2010~1000 2024-03-07T03:14:13.202 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:14:13.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:12 smithi145 bash[30906]: cluster 2024-03-07T03:14:11.674224+0000 mgr.y (mgr.15132) 12 : cluster 0 pgmap v22: 137 pgs: 137 active+clean; 741 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 7.4 MiB/s wr, 342 op/s 2024-03-07T03:14:13.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:12 smithi145 bash[30906]: cluster 2024-03-07T03:14:11.831218+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2024-03-07T03:14:13.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:12 smithi145 bash[30906]: audit 2024-03-07T03:14:12.805714+0000 mon.a (mon.0) 1196 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:13.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:12 smithi145 bash[21339]: cluster 2024-03-07T03:14:11.674224+0000 mgr.y (mgr.15132) 12 : cluster 0 pgmap v22: 137 pgs: 137 active+clean; 741 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 7.4 MiB/s wr, 342 op/s 2024-03-07T03:14:13.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:12 smithi145 bash[21339]: cluster 2024-03-07T03:14:11.831218+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2024-03-07T03:14:13.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:12 smithi145 bash[21339]: audit 2024-03-07T03:14:12.805714+0000 mon.a (mon.0) 1196 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:13.822 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:14:13.824 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:13 smithi189 bash[58488]: ts=2024-03-07T03:14:13.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:14:14.079 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:14:14.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:13 smithi189 bash[23614]: cluster 2024-03-07T03:14:12.833194+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2024-03-07T03:14:14.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:13 smithi189 bash[23614]: cluster 2024-03-07T03:14:13.473709+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2024-03-07T03:14:14.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:13 smithi145 bash[21339]: cluster 2024-03-07T03:14:12.833194+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2024-03-07T03:14:14.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:13 smithi145 bash[21339]: cluster 2024-03-07T03:14:13.473709+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2024-03-07T03:14:14.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:13 smithi145 bash[30906]: cluster 2024-03-07T03:14:12.833194+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2024-03-07T03:14:14.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:13 smithi145 bash[30906]: cluster 2024-03-07T03:14:13.473709+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2024-03-07T03:14:14.318 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.Clone (5801 ms) 2024-03-07T03:14:14.318 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.CloneParent 2024-03-07T03:14:14.360 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:14:14.377 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap 2024-03-07T03:14:15.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:14 smithi189 bash[23614]: cluster 2024-03-07T03:14:13.674755+0000 mgr.y (mgr.15132) 13 : cluster 0 pgmap v26: 137 pgs: 137 active+clean; 741 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:15.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:14 smithi189 bash[23614]: audit 2024-03-07T03:14:14.089041+0000 mon.a (mon.0) 1199 : audit 0 from='client.? 172.21.15.145:0/381097027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:15.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:14 smithi145 bash[21339]: cluster 2024-03-07T03:14:13.674755+0000 mgr.y (mgr.15132) 13 : cluster 0 pgmap v26: 137 pgs: 137 active+clean; 741 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:15.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:14 smithi145 bash[21339]: audit 2024-03-07T03:14:14.089041+0000 mon.a (mon.0) 1199 : audit 0 from='client.? 172.21.15.145:0/381097027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:15.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:14 smithi145 bash[30906]: cluster 2024-03-07T03:14:13.674755+0000 mgr.y (mgr.15132) 13 : cluster 0 pgmap v26: 137 pgs: 137 active+clean; 741 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:15.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:14 smithi145 bash[30906]: audit 2024-03-07T03:14:14.089041+0000 mon.a (mon.0) 1199 : audit 0 from='client.? 172.21.15.145:0/381097027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:15.855 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap 2024-03-07T03:14:15.869 INFO:tasks.workunit.client.0.smithi145.stdout:clone image111 -> image114 2024-03-07T03:14:15.991 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:14:16.195 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:14:16.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:15 smithi145 bash[21339]: cluster 2024-03-07T03:14:14.856888+0000 mon.a (mon.0) 1200 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2024-03-07T03:14:16.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:15 smithi145 bash[21339]: audit 2024-03-07T03:14:15.529731+0000 mon.a (mon.0) 1201 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:16.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:15 smithi145 bash[21339]: audit 2024-03-07T03:14:15.542151+0000 mon.a (mon.0) 1202 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:16.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:15 smithi145 bash[21339]: audit 2024-03-07T03:14:15.545398+0000 mon.a (mon.0) 1203 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:16.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:15 smithi145 bash[21339]: audit 2024-03-07T03:14:15.547209+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:14:16.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:15 smithi145 bash[30906]: cluster 2024-03-07T03:14:14.856888+0000 mon.a (mon.0) 1200 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2024-03-07T03:14:16.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:15 smithi145 bash[30906]: audit 2024-03-07T03:14:15.529731+0000 mon.a (mon.0) 1201 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:16.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:15 smithi145 bash[30906]: audit 2024-03-07T03:14:15.542151+0000 mon.a (mon.0) 1202 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:16.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:15 smithi145 bash[30906]: audit 2024-03-07T03:14:15.545398+0000 mon.a (mon.0) 1203 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:16.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:15 smithi145 bash[30906]: audit 2024-03-07T03:14:15.547209+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:14:16.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:15 smithi189 bash[23614]: cluster 2024-03-07T03:14:14.856888+0000 mon.a (mon.0) 1200 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2024-03-07T03:14:16.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:15 smithi189 bash[23614]: audit 2024-03-07T03:14:15.529731+0000 mon.a (mon.0) 1201 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:16.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:15 smithi189 bash[23614]: audit 2024-03-07T03:14:15.542151+0000 mon.a (mon.0) 1202 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:16.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:15 smithi189 bash[23614]: audit 2024-03-07T03:14:15.545398+0000 mon.a (mon.0) 1203 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:16.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:15 smithi189 bash[23614]: audit 2024-03-07T03:14:15.547209+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:14:16.366 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.CloneParent (2048 ms) 2024-03-07T03:14:16.366 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.CloneUpdateAfterPrepare 2024-03-07T03:14:16.402 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:14:16.416 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 0~10 2024-03-07T03:14:16.426 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap 2024-03-07T03:14:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: cephadm 2024-03-07T03:14:15.549874+0000 mgr.y (mgr.15132) 14 : cephadm 1 Updating smithi145:/etc/ceph/ceph.conf 2024-03-07T03:14:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: cephadm 2024-03-07T03:14:15.550283+0000 mgr.y (mgr.15132) 15 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T03:14:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: cluster 2024-03-07T03:14:15.675237+0000 mgr.y (mgr.15132) 16 : cluster 0 pgmap v28: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 9.0 MiB/s wr, 463 op/s 2024-03-07T03:14:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: cephadm 2024-03-07T03:14:15.815674+0000 mgr.y (mgr.15132) 17 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:14:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: cephadm 2024-03-07T03:14:15.816329+0000 mgr.y (mgr.15132) 18 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:14:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: cluster 2024-03-07T03:14:15.856512+0000 mon.a (mon.0) 1205 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2024-03-07T03:14:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: cephadm 2024-03-07T03:14:15.980205+0000 mgr.y (mgr.15132) 19 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:14:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: cephadm 2024-03-07T03:14:15.980773+0000 mgr.y (mgr.15132) 20 : cephadm 1 Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:14:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: cephadm 2024-03-07T03:14:16.145004+0000 mgr.y (mgr.15132) 21 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:14:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: cephadm 2024-03-07T03:14:16.145251+0000 mgr.y (mgr.15132) 22 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:14:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: audit 2024-03-07T03:14:16.316756+0000 mon.a (mon.0) 1206 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: audit 2024-03-07T03:14:16.320508+0000 mon.a (mon.0) 1207 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: audit 2024-03-07T03:14:16.326893+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: audit 2024-03-07T03:14:16.335516+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: audit 2024-03-07T03:14:16.345476+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: cephadm 2024-03-07T03:14:16.400323+0000 mgr.y (mgr.15132) 23 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:14:17.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[21339]: audit 2024-03-07T03:14:16.565958+0000 mon.a (mon.0) 1211 : audit 0 from='client.? 172.21.15.145:0/2887243240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:17.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: cephadm 2024-03-07T03:14:15.549874+0000 mgr.y (mgr.15132) 14 : cephadm 1 Updating smithi145:/etc/ceph/ceph.conf 2024-03-07T03:14:17.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: cephadm 2024-03-07T03:14:15.550283+0000 mgr.y (mgr.15132) 15 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T03:14:17.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: cluster 2024-03-07T03:14:15.675237+0000 mgr.y (mgr.15132) 16 : cluster 0 pgmap v28: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 9.0 MiB/s wr, 463 op/s 2024-03-07T03:14:17.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: cephadm 2024-03-07T03:14:15.815674+0000 mgr.y (mgr.15132) 17 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:14:17.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: cephadm 2024-03-07T03:14:15.816329+0000 mgr.y (mgr.15132) 18 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:14:17.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: cluster 2024-03-07T03:14:15.856512+0000 mon.a (mon.0) 1205 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2024-03-07T03:14:17.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: cephadm 2024-03-07T03:14:15.980205+0000 mgr.y (mgr.15132) 19 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:14:17.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: cephadm 2024-03-07T03:14:15.980773+0000 mgr.y (mgr.15132) 20 : cephadm 1 Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:14:17.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: cephadm 2024-03-07T03:14:16.145004+0000 mgr.y (mgr.15132) 21 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:14:17.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: cephadm 2024-03-07T03:14:16.145251+0000 mgr.y (mgr.15132) 22 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:14:17.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: audit 2024-03-07T03:14:16.316756+0000 mon.a (mon.0) 1206 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: audit 2024-03-07T03:14:16.320508+0000 mon.a (mon.0) 1207 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: audit 2024-03-07T03:14:16.326893+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: audit 2024-03-07T03:14:16.335516+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: audit 2024-03-07T03:14:16.345476+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: cephadm 2024-03-07T03:14:16.400323+0000 mgr.y (mgr.15132) 23 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:14:17.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:16 smithi145 bash[30906]: audit 2024-03-07T03:14:16.565958+0000 mon.a (mon.0) 1211 : audit 0 from='client.? 172.21.15.145:0/2887243240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:17.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: cephadm 2024-03-07T03:14:15.549874+0000 mgr.y (mgr.15132) 14 : cephadm 1 Updating smithi145:/etc/ceph/ceph.conf 2024-03-07T03:14:17.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: cephadm 2024-03-07T03:14:15.550283+0000 mgr.y (mgr.15132) 15 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T03:14:17.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: cluster 2024-03-07T03:14:15.675237+0000 mgr.y (mgr.15132) 16 : cluster 0 pgmap v28: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 9.0 MiB/s wr, 463 op/s 2024-03-07T03:14:17.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: cephadm 2024-03-07T03:14:15.815674+0000 mgr.y (mgr.15132) 17 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:14:17.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: cephadm 2024-03-07T03:14:15.816329+0000 mgr.y (mgr.15132) 18 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:14:17.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: cluster 2024-03-07T03:14:15.856512+0000 mon.a (mon.0) 1205 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2024-03-07T03:14:17.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: cephadm 2024-03-07T03:14:15.980205+0000 mgr.y (mgr.15132) 19 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:14:17.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: cephadm 2024-03-07T03:14:15.980773+0000 mgr.y (mgr.15132) 20 : cephadm 1 Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:14:17.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: cephadm 2024-03-07T03:14:16.145004+0000 mgr.y (mgr.15132) 21 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:14:17.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: cephadm 2024-03-07T03:14:16.145251+0000 mgr.y (mgr.15132) 22 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:14:17.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: audit 2024-03-07T03:14:16.316756+0000 mon.a (mon.0) 1206 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: audit 2024-03-07T03:14:16.320508+0000 mon.a (mon.0) 1207 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: audit 2024-03-07T03:14:16.326893+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: audit 2024-03-07T03:14:16.335516+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: audit 2024-03-07T03:14:16.345476+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:17.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: cephadm 2024-03-07T03:14:16.400323+0000 mgr.y (mgr.15132) 23 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:14:17.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:16 smithi189 bash[23614]: audit 2024-03-07T03:14:16.565958+0000 mon.a (mon.0) 1211 : audit 0 from='client.? 172.21.15.145:0/2887243240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:18.372 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap 2024-03-07T03:14:18.392 INFO:tasks.workunit.client.0.smithi145.stdout:clone image115 -> image118 2024-03-07T03:14:18.468 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:18 smithi189 bash[23614]: cephadm 2024-03-07T03:14:16.642894+0000 mgr.y (mgr.15132) 24 : cephadm 1 Reconfiguring daemon prometheus.a on smithi189 2024-03-07T03:14:18.468 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:18 smithi189 bash[23614]: cluster 2024-03-07T03:14:17.355882+0000 mon.a (mon.0) 1212 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2024-03-07T03:14:18.623 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:14:18.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:18 smithi145 bash[21339]: cephadm 2024-03-07T03:14:16.642894+0000 mgr.y (mgr.15132) 24 : cephadm 1 Reconfiguring daemon prometheus.a on smithi189 2024-03-07T03:14:18.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:18 smithi145 bash[21339]: cluster 2024-03-07T03:14:17.355882+0000 mon.a (mon.0) 1212 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2024-03-07T03:14:18.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:18 smithi145 bash[30906]: cephadm 2024-03-07T03:14:16.642894+0000 mgr.y (mgr.15132) 24 : cephadm 1 Reconfiguring daemon prometheus.a on smithi189 2024-03-07T03:14:18.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:18 smithi145 bash[30906]: cluster 2024-03-07T03:14:17.355882+0000 mon.a (mon.0) 1212 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2024-03-07T03:14:18.871 INFO:tasks.workunit.client.0.smithi145.stdout:write: Y 0~1 2024-03-07T03:14:18.919 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:14:19.074 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:14:19.297 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.CloneUpdateAfterPrepare (2931 ms) 2024-03-07T03:14:19.297 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.TriggerAssertSnapcSeq 2024-03-07T03:14:19.382 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:14:19.411 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 41943040~10 2024-03-07T03:14:19.433 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:14:19.724 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:19 smithi189 bash[23614]: cluster 2024-03-07T03:14:17.675975+0000 mgr.y (mgr.15132) 25 : cluster 0 pgmap v31: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 440 op/s 2024-03-07T03:14:19.724 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:19 smithi189 bash[23614]: cluster 2024-03-07T03:14:18.368409+0000 mon.a (mon.0) 1213 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2024-03-07T03:14:19.724 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:19 smithi189 bash[23614]: audit 2024-03-07T03:14:19.045071+0000 mon.c (mon.2) 173 : audit 0 from='client.? 172.21.15.145:0/3412792300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:19.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:19 smithi145 bash[21339]: cluster 2024-03-07T03:14:17.675975+0000 mgr.y (mgr.15132) 25 : cluster 0 pgmap v31: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 440 op/s 2024-03-07T03:14:19.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:19 smithi145 bash[21339]: cluster 2024-03-07T03:14:18.368409+0000 mon.a (mon.0) 1213 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2024-03-07T03:14:19.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:19 smithi145 bash[21339]: audit 2024-03-07T03:14:19.045071+0000 mon.c (mon.2) 173 : audit 0 from='client.? 172.21.15.145:0/3412792300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:19.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:19 smithi145 bash[30906]: cluster 2024-03-07T03:14:17.675975+0000 mgr.y (mgr.15132) 25 : cluster 0 pgmap v31: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 440 op/s 2024-03-07T03:14:19.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:19 smithi145 bash[30906]: cluster 2024-03-07T03:14:18.368409+0000 mon.a (mon.0) 1213 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2024-03-07T03:14:19.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:19 smithi145 bash[30906]: audit 2024-03-07T03:14:19.045071+0000 mon.c (mon.2) 173 : audit 0 from='client.? 172.21.15.145:0/3412792300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:20.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:20 smithi189 bash[52103]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:14:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-07T03:14:20.810 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:20 smithi189 systemd[1]: Stopping Ceph prometheus.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:14:20.811 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:20 smithi189 bash[58488]: ts=2024-03-07T03:14:20.659Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-07T03:14:20.811 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:20 smithi189 bash[58488]: ts=2024-03-07T03:14:20.659Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-07T03:14:20.811 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:20 smithi189 bash[58488]: ts=2024-03-07T03:14:20.659Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-07T03:14:20.811 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:20 smithi189 bash[58488]: ts=2024-03-07T03:14:20.659Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-07T03:14:20.811 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:20 smithi189 bash[58488]: ts=2024-03-07T03:14:20.659Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-07T03:14:20.811 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:20 smithi189 bash[58488]: ts=2024-03-07T03:14:20.659Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-07T03:14:20.811 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:20 smithi189 bash[58488]: ts=2024-03-07T03:14:20.659Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-07T03:14:20.811 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:20 smithi189 bash[58488]: ts=2024-03-07T03:14:20.659Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-07T03:14:20.811 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:20 smithi189 bash[58488]: ts=2024-03-07T03:14:20.659Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-07T03:14:20.811 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:20 smithi189 bash[58488]: ts=2024-03-07T03:14:20.660Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-07T03:14:20.811 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:20 smithi189 bash[58488]: ts=2024-03-07T03:14:20.660Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-07T03:14:20.811 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:20 smithi189 bash[58488]: ts=2024-03-07T03:14:20.660Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-07T03:14:21.080 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:21 smithi189 bash[59722]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-prometheus-a 2024-03-07T03:14:21.385 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 41943041~10 2024-03-07T03:14:21.395 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:14:21.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:21 smithi189 bash[59785]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-prometheus-a 2024-03-07T03:14:21.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:21 smithi189 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-07T03:14:21.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:21 smithi189 systemd[1]: Stopped Ceph prometheus.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:14:21.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:21 smithi189 systemd[1]: Started Ceph prometheus.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:14:21.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:21 smithi189 bash[23614]: cluster 2024-03-07T03:14:19.677563+0000 mgr.y (mgr.15132) 26 : cluster 0 pgmap v33: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 874 KiB/s wr, 724 op/s 2024-03-07T03:14:21.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:21 smithi189 bash[23614]: cluster 2024-03-07T03:14:20.375246+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2024-03-07T03:14:21.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:21 smithi189 bash[23614]: audit 2024-03-07T03:14:21.335164+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:21.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:21 smithi189 bash[23614]: audit 2024-03-07T03:14:21.346308+0000 mon.a (mon.0) 1216 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:21.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:21 smithi145 bash[21339]: cluster 2024-03-07T03:14:19.677563+0000 mgr.y (mgr.15132) 26 : cluster 0 pgmap v33: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 874 KiB/s wr, 724 op/s 2024-03-07T03:14:21.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:21 smithi145 bash[21339]: cluster 2024-03-07T03:14:20.375246+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2024-03-07T03:14:21.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:21 smithi145 bash[21339]: audit 2024-03-07T03:14:21.335164+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:21.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:21 smithi145 bash[21339]: audit 2024-03-07T03:14:21.346308+0000 mon.a (mon.0) 1216 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:21.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:21 smithi145 bash[30906]: cluster 2024-03-07T03:14:19.677563+0000 mgr.y (mgr.15132) 26 : cluster 0 pgmap v33: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 874 KiB/s wr, 724 op/s 2024-03-07T03:14:21.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:21 smithi145 bash[30906]: cluster 2024-03-07T03:14:20.375246+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2024-03-07T03:14:21.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:21 smithi145 bash[30906]: audit 2024-03-07T03:14:21.335164+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:21.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:21 smithi145 bash[30906]: audit 2024-03-07T03:14:21.346308+0000 mon.a (mon.0) 1216 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:22.700 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 41943042~10 2024-03-07T03:14:22.738 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:14:22.751 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[23614]: cluster 2024-03-07T03:14:21.377489+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2024-03-07T03:14:22.751 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[23614]: audit 2024-03-07T03:14:21.391201+0000 mon.a (mon.0) 1218 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:14:22.751 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[23614]: audit 2024-03-07T03:14:21.392658+0000 mgr.y (mgr.15132) 27 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:14:22.752 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[23614]: audit 2024-03-07T03:14:21.413051+0000 mon.a (mon.0) 1219 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:22.752 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[23614]: audit 2024-03-07T03:14:21.507337+0000 mon.c (mon.2) 174 : audit 0 from='client.? 172.21.15.145:0/2767689059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:22.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:22 smithi145 bash[21339]: cluster 2024-03-07T03:14:21.377489+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2024-03-07T03:14:22.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:22 smithi145 bash[21339]: audit 2024-03-07T03:14:21.391201+0000 mon.a (mon.0) 1218 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:14:22.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:22 smithi145 bash[21339]: audit 2024-03-07T03:14:21.392658+0000 mgr.y (mgr.15132) 27 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:14:22.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:22 smithi145 bash[21339]: audit 2024-03-07T03:14:21.413051+0000 mon.a (mon.0) 1219 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:22.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:22 smithi145 bash[21339]: audit 2024-03-07T03:14:21.507337+0000 mon.c (mon.2) 174 : audit 0 from='client.? 172.21.15.145:0/2767689059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:22.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:22 smithi145 bash[30906]: cluster 2024-03-07T03:14:21.377489+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2024-03-07T03:14:22.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:22 smithi145 bash[30906]: audit 2024-03-07T03:14:21.391201+0000 mon.a (mon.0) 1218 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:14:22.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:22 smithi145 bash[30906]: audit 2024-03-07T03:14:21.392658+0000 mgr.y (mgr.15132) 27 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:14:22.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:22 smithi145 bash[30906]: audit 2024-03-07T03:14:21.413051+0000 mon.a (mon.0) 1219 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:22.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:22 smithi145 bash[30906]: audit 2024-03-07T03:14:21.507337+0000 mon.c (mon.2) 174 : audit 0 from='client.? 172.21.15.145:0/2767689059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:23.031 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.882Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-07T03:14:23.031 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.882Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-07T03:14:23.031 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.882Z caller=main.go:570 level=info host_details="(Linux 5.15.0-97-generic #107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024 x86_64 smithi189 (none))" 2024-03-07T03:14:23.031 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.882Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.882Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.884Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.884Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.886Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.886Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.889Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.889Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.957µs 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.889Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.891Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.907Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.914Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.914Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.915Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.422µs wal_replay_duration=25.635579ms wbl_replay_duration=137ns total_replay_duration=25.680363ms 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.917Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-07T03:14:23.032 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.917Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-07T03:14:23.033 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.917Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-07T03:14:23.033 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.943Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.867013ms db_storage=1.051µs remote_storage=1.262µs web_handler=509ns query_engine=1.148µs scrape=278.894µs scrape_sd=103µs notify=24.505µs notify_sd=20.787µs rules=24.784738ms tracing=7.763µs 2024-03-07T03:14:23.033 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.943Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-07T03:14:23.033 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:22 smithi189 bash[59814]: ts=2024-03-07T03:14:22.943Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-07T03:14:23.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:23 smithi145 bash[30906]: cluster 2024-03-07T03:14:21.678332+0000 mgr.y (mgr.15132) 28 : cluster 0 pgmap v36: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 974 KiB/s wr, 807 op/s 2024-03-07T03:14:23.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:23 smithi145 bash[30906]: cluster 2024-03-07T03:14:22.396296+0000 mon.a (mon.0) 1220 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2024-03-07T03:14:23.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:23 smithi145 bash[21339]: cluster 2024-03-07T03:14:21.678332+0000 mgr.y (mgr.15132) 28 : cluster 0 pgmap v36: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 974 KiB/s wr, 807 op/s 2024-03-07T03:14:23.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:23 smithi145 bash[21339]: cluster 2024-03-07T03:14:22.396296+0000 mon.a (mon.0) 1220 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2024-03-07T03:14:23.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:23 smithi189 bash[23614]: cluster 2024-03-07T03:14:21.678332+0000 mgr.y (mgr.15132) 28 : cluster 0 pgmap v36: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 974 KiB/s wr, 807 op/s 2024-03-07T03:14:23.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:23 smithi189 bash[23614]: cluster 2024-03-07T03:14:22.396296+0000 mon.a (mon.0) 1220 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2024-03-07T03:14:23.831 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:23 smithi189 bash[59814]: ts=2024-03-07T03:14:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:14:24.436 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 41943040 2024-03-07T03:14:24.503 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:14:24.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:24 smithi189 bash[23614]: cluster 2024-03-07T03:14:23.396836+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2024-03-07T03:14:24.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:24 smithi189 bash[23614]: audit 2024-03-07T03:14:23.739262+0000 mon.a (mon.0) 1222 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:14:24.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:24 smithi189 bash[23614]: audit 2024-03-07T03:14:23.741128+0000 mon.a (mon.0) 1223 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:14:24.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:24 smithi189 bash[23614]: audit 2024-03-07T03:14:23.742891+0000 mon.a (mon.0) 1224 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:24.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:24 smithi189 bash[23614]: audit 2024-03-07T03:14:23.952876+0000 mon.a (mon.0) 1225 : audit 0 from='client.? 172.21.15.145:0/2127673455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:24.663 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:14:24.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:24 smithi145 bash[21339]: cluster 2024-03-07T03:14:23.396836+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2024-03-07T03:14:24.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:24 smithi145 bash[21339]: audit 2024-03-07T03:14:23.739262+0000 mon.a (mon.0) 1222 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:14:24.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:24 smithi145 bash[21339]: audit 2024-03-07T03:14:23.741128+0000 mon.a (mon.0) 1223 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:14:24.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:24 smithi145 bash[21339]: audit 2024-03-07T03:14:23.742891+0000 mon.a (mon.0) 1224 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:24.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:24 smithi145 bash[21339]: audit 2024-03-07T03:14:23.952876+0000 mon.a (mon.0) 1225 : audit 0 from='client.? 172.21.15.145:0/2127673455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:24.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:24 smithi145 bash[30906]: cluster 2024-03-07T03:14:23.396836+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2024-03-07T03:14:24.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:24 smithi145 bash[30906]: audit 2024-03-07T03:14:23.739262+0000 mon.a (mon.0) 1222 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:14:24.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:24 smithi145 bash[30906]: audit 2024-03-07T03:14:23.741128+0000 mon.a (mon.0) 1223 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:14:24.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:24 smithi145 bash[30906]: audit 2024-03-07T03:14:23.742891+0000 mon.a (mon.0) 1224 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:24.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:24 smithi145 bash[30906]: audit 2024-03-07T03:14:23.952876+0000 mon.a (mon.0) 1225 : audit 0 from='client.? 172.21.15.145:0/2127673455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:25.722 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.TriggerAssertSnapcSeq (6425 ms) 2024-03-07T03:14:25.722 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.SnapTrimBeforePrepare 2024-03-07T03:14:25.801 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:14:25.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:25 smithi145 bash[21339]: cluster 2024-03-07T03:14:23.678793+0000 mgr.y (mgr.15132) 29 : cluster 0 pgmap v39: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:25.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:25 smithi145 bash[21339]: cephadm 2024-03-07T03:14:23.738460+0000 mgr.y (mgr.15132) 30 : cephadm 1 Upgrade: Updating mgr.x 2024-03-07T03:14:25.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:25 smithi145 bash[21339]: cephadm 2024-03-07T03:14:23.744713+0000 mgr.y (mgr.15132) 31 : cephadm 1 Deploying daemon mgr.x on smithi189 2024-03-07T03:14:25.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:25 smithi145 bash[21339]: cluster 2024-03-07T03:14:24.415043+0000 mon.a (mon.0) 1226 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2024-03-07T03:14:25.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:25 smithi145 bash[30906]: cluster 2024-03-07T03:14:23.678793+0000 mgr.y (mgr.15132) 29 : cluster 0 pgmap v39: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:25.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:25 smithi145 bash[30906]: cephadm 2024-03-07T03:14:23.738460+0000 mgr.y (mgr.15132) 30 : cephadm 1 Upgrade: Updating mgr.x 2024-03-07T03:14:25.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:25 smithi145 bash[30906]: cephadm 2024-03-07T03:14:23.744713+0000 mgr.y (mgr.15132) 31 : cephadm 1 Deploying daemon mgr.x on smithi189 2024-03-07T03:14:25.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:25 smithi145 bash[30906]: cluster 2024-03-07T03:14:24.415043+0000 mon.a (mon.0) 1226 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2024-03-07T03:14:25.830 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 41943040~10 2024-03-07T03:14:25.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:25 smithi189 bash[23614]: cluster 2024-03-07T03:14:23.678793+0000 mgr.y (mgr.15132) 29 : cluster 0 pgmap v39: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:25.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:25 smithi189 bash[23614]: cephadm 2024-03-07T03:14:23.738460+0000 mgr.y (mgr.15132) 30 : cephadm 1 Upgrade: Updating mgr.x 2024-03-07T03:14:25.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:25 smithi189 bash[23614]: cephadm 2024-03-07T03:14:23.744713+0000 mgr.y (mgr.15132) 31 : cephadm 1 Deploying daemon mgr.x on smithi189 2024-03-07T03:14:25.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:25 smithi189 bash[23614]: cluster 2024-03-07T03:14:24.415043+0000 mon.a (mon.0) 1226 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2024-03-07T03:14:25.853 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:14:26.233 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:26 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:26.233 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:14:26 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:26.234 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:14:26 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:26.234 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:14:26 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:26.234 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:26 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:26.234 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:14:26 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:26.235 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:14:26 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:26.235 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:14:26 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:26.235 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:26 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:26.235 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:26 smithi189 systemd[1]: Stopping Ceph mgr.x for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:14:26.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:26 smithi189 bash[23614]: cluster 2024-03-07T03:14:25.484397+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2024-03-07T03:14:26.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:26 smithi189 bash[23614]: cluster 2024-03-07T03:14:25.679677+0000 mgr.y (mgr.15132) 32 : cluster 0 pgmap v42: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.0 MiB/s wr, 376 op/s 2024-03-07T03:14:26.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:26 smithi189 bash[23614]: audit 2024-03-07T03:14:26.389259+0000 mon.a (mon.0) 1228 : audit 0 from='client.? 172.21.15.145:0/673616889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:26.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:26 smithi145 bash[21339]: cluster 2024-03-07T03:14:25.484397+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2024-03-07T03:14:26.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:26 smithi145 bash[21339]: cluster 2024-03-07T03:14:25.679677+0000 mgr.y (mgr.15132) 32 : cluster 0 pgmap v42: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.0 MiB/s wr, 376 op/s 2024-03-07T03:14:26.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:26 smithi145 bash[21339]: audit 2024-03-07T03:14:26.389259+0000 mon.a (mon.0) 1228 : audit 0 from='client.? 172.21.15.145:0/673616889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:26.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:26 smithi145 bash[30906]: cluster 2024-03-07T03:14:25.484397+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2024-03-07T03:14:26.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:26 smithi145 bash[30906]: cluster 2024-03-07T03:14:25.679677+0000 mgr.y (mgr.15132) 32 : cluster 0 pgmap v42: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.0 MiB/s wr, 376 op/s 2024-03-07T03:14:26.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:26 smithi145 bash[30906]: audit 2024-03-07T03:14:26.389259+0000 mon.a (mon.0) 1228 : audit 0 from='client.? 172.21.15.145:0/673616889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:27.284 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:26 smithi189 bash[60092]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mgr-x 2024-03-07T03:14:27.284 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:27 smithi189 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-03-07T03:14:27.284 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:27 smithi189 bash[60151]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mgr-x 2024-03-07T03:14:27.284 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:27 smithi189 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.x.service: Failed with result 'exit-code'. 2024-03-07T03:14:27.284 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:27 smithi189 systemd[1]: Stopped Ceph mgr.x for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:14:27.499 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 41943040 2024-03-07T03:14:27.542 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:14:27.580 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:14:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:27.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:27.581 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:14:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:27.581 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:14:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:27.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:27.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:27 smithi189 bash[23614]: cluster 2024-03-07T03:14:26.489548+0000 mon.a (mon.0) 1229 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2024-03-07T03:14:27.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:27 smithi189 bash[23614]: audit 2024-03-07T03:14:27.466114+0000 mon.a (mon.0) 1230 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:27.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:27 smithi189 bash[23614]: audit 2024-03-07T03:14:27.479719+0000 mon.a (mon.0) 1231 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:27.582 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:14:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:27.582 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:27.582 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:27 smithi189 systemd[1]: Started Ceph mgr.x for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:14:27.582 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:14:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:27.583 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:14:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:27.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:27 smithi145 bash[21339]: cluster 2024-03-07T03:14:26.489548+0000 mon.a (mon.0) 1229 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2024-03-07T03:14:27.826 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:27 smithi145 bash[21339]: audit 2024-03-07T03:14:27.466114+0000 mon.a (mon.0) 1230 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:27.826 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:27 smithi145 bash[21339]: audit 2024-03-07T03:14:27.479719+0000 mon.a (mon.0) 1231 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:27.827 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:27 smithi145 bash[30906]: cluster 2024-03-07T03:14:26.489548+0000 mon.a (mon.0) 1229 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2024-03-07T03:14:27.827 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:27 smithi145 bash[30906]: audit 2024-03-07T03:14:27.466114+0000 mon.a (mon.0) 1230 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:27.827 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:27 smithi145 bash[30906]: audit 2024-03-07T03:14:27.479719+0000 mon.a (mon.0) 1231 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:28.554 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:28 smithi189 bash[60207]: debug 2024-03-07T03:14:28.397+0000 7feca5a7e700 1 -- 172.21.15.189:0/3354689440 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5643b0a9a5a0 con 0x5643b181e400 2024-03-07T03:14:28.554 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:28 smithi189 bash[23614]: cluster 2024-03-07T03:14:27.492382+0000 mon.a (mon.0) 1232 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2024-03-07T03:14:28.554 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:28 smithi189 bash[23614]: audit 2024-03-07T03:14:27.518938+0000 mon.a (mon.0) 1233 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:28.554 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:28 smithi189 bash[23614]: cluster 2024-03-07T03:14:27.680427+0000 mgr.y (mgr.15132) 33 : cluster 0 pgmap v45: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.0 MiB/s wr, 376 op/s 2024-03-07T03:14:28.554 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:28 smithi189 bash[23614]: audit 2024-03-07T03:14:27.805942+0000 mon.a (mon.0) 1234 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:28.670 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:14:28.694 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:14:28.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:28 smithi145 bash[21339]: cluster 2024-03-07T03:14:27.492382+0000 mon.a (mon.0) 1232 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2024-03-07T03:14:28.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:28 smithi145 bash[21339]: audit 2024-03-07T03:14:27.518938+0000 mon.a (mon.0) 1233 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:28.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:28 smithi145 bash[21339]: cluster 2024-03-07T03:14:27.680427+0000 mgr.y (mgr.15132) 33 : cluster 0 pgmap v45: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.0 MiB/s wr, 376 op/s 2024-03-07T03:14:28.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:28 smithi145 bash[21339]: audit 2024-03-07T03:14:27.805942+0000 mon.a (mon.0) 1234 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:28.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:28 smithi145 bash[30906]: cluster 2024-03-07T03:14:27.492382+0000 mon.a (mon.0) 1232 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2024-03-07T03:14:28.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:28 smithi145 bash[30906]: audit 2024-03-07T03:14:27.518938+0000 mon.a (mon.0) 1233 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:28.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:28 smithi145 bash[30906]: cluster 2024-03-07T03:14:27.680427+0000 mgr.y (mgr.15132) 33 : cluster 0 pgmap v45: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.0 MiB/s wr, 376 op/s 2024-03-07T03:14:28.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:28 smithi145 bash[30906]: audit 2024-03-07T03:14:27.805942+0000 mon.a (mon.0) 1234 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:28.830 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:28 smithi189 bash[60207]: debug 2024-03-07T03:14:28.561+0000 7fecafc80200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:14:29.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:29 smithi145 bash[21339]: cluster 2024-03-07T03:14:28.506759+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2024-03-07T03:14:29.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:29 smithi145 bash[21339]: audit 2024-03-07T03:14:28.860634+0000 mon.a (mon.0) 1236 : audit 0 from='client.? 172.21.15.145:0/1602135505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:29.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:29 smithi145 bash[30906]: cluster 2024-03-07T03:14:28.506759+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2024-03-07T03:14:29.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:29 smithi145 bash[30906]: audit 2024-03-07T03:14:28.860634+0000 mon.a (mon.0) 1236 : audit 0 from='client.? 172.21.15.145:0/1602135505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:29.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:29 smithi189 bash[23614]: cluster 2024-03-07T03:14:28.506759+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2024-03-07T03:14:29.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:29 smithi189 bash[23614]: audit 2024-03-07T03:14:28.860634+0000 mon.a (mon.0) 1236 : audit 0 from='client.? 172.21.15.145:0/1602135505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:30.513 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:30 smithi189 bash[60207]: debug 2024-03-07T03:14:30.169+0000 7fecafc80200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:14:30.513 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:30 smithi189 bash[60207]: debug 2024-03-07T03:14:30.269+0000 7fecafc80200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:14:30.532 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 41943040~10 2024-03-07T03:14:30.540 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap4 2024-03-07T03:14:30.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:30 smithi145 bash[21339]: cluster 2024-03-07T03:14:29.511126+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2024-03-07T03:14:30.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:30 smithi145 bash[21339]: cluster 2024-03-07T03:14:29.681241+0000 mgr.y (mgr.15132) 34 : cluster 0 pgmap v48: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 277 KiB/s rd, 25 KiB/s wr, 373 op/s 2024-03-07T03:14:30.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:30 smithi145 bash[30906]: cluster 2024-03-07T03:14:29.511126+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2024-03-07T03:14:30.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:30 smithi145 bash[30906]: cluster 2024-03-07T03:14:29.681241+0000 mgr.y (mgr.15132) 34 : cluster 0 pgmap v48: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 277 KiB/s rd, 25 KiB/s wr, 373 op/s 2024-03-07T03:14:30.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:30 smithi189 bash[23614]: cluster 2024-03-07T03:14:29.511126+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2024-03-07T03:14:30.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:30 smithi189 bash[23614]: cluster 2024-03-07T03:14:29.681241+0000 mgr.y (mgr.15132) 34 : cluster 0 pgmap v48: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 277 KiB/s rd, 25 KiB/s wr, 373 op/s 2024-03-07T03:14:30.830 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:30 smithi189 bash[60207]: debug 2024-03-07T03:14:30.521+0000 7fecafc80200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:14:31.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:31 smithi189 bash[23614]: cluster 2024-03-07T03:14:30.530988+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2024-03-07T03:14:31.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:31 smithi189 bash[23614]: audit 2024-03-07T03:14:31.303751+0000 mon.c (mon.2) 175 : audit 0 from='client.? 172.21.15.145:0/110822441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:31.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:31 smithi189 bash[60207]: debug 2024-03-07T03:14:31.325+0000 7fecafc80200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:14:31.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:31 smithi189 bash[60207]: debug 2024-03-07T03:14:31.437+0000 7fecafc80200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:14:31.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:31 smithi189 bash[60207]: debug 2024-03-07T03:14:31.545+0000 7fecafc80200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:14:31.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:31 smithi145 bash[21339]: cluster 2024-03-07T03:14:30.530988+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2024-03-07T03:14:31.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:31 smithi145 bash[21339]: audit 2024-03-07T03:14:31.303751+0000 mon.c (mon.2) 175 : audit 0 from='client.? 172.21.15.145:0/110822441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:31.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:31 smithi145 bash[30906]: cluster 2024-03-07T03:14:30.530988+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2024-03-07T03:14:31.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:31 smithi145 bash[30906]: audit 2024-03-07T03:14:31.303751+0000 mon.c (mon.2) 175 : audit 0 from='client.? 172.21.15.145:0/110822441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:32.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:31 smithi189 bash[60207]: debug 2024-03-07T03:14:31.905+0000 7fecafc80200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:14:32.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:32 smithi189 bash[60207]: debug 2024-03-07T03:14:32.025+0000 7fecafc80200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:14:32.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:32 smithi189 bash[60207]: debug 2024-03-07T03:14:32.129+0000 7fecafc80200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:14:32.560 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 41943040 2024-03-07T03:14:32.602 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:14:32.710 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:14:32.763 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:32 smithi189 bash[60207]: debug 2024-03-07T03:14:32.489+0000 7fecafc80200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:14:32.763 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:32 smithi189 bash[23614]: cluster 2024-03-07T03:14:31.531697+0000 mon.a (mon.0) 1239 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2024-03-07T03:14:32.763 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:32 smithi189 bash[23614]: cluster 2024-03-07T03:14:31.682068+0000 mgr.y (mgr.15132) 35 : cluster 0 pgmap v51: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 277 KiB/s rd, 25 KiB/s wr, 373 op/s 2024-03-07T03:14:32.808 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:14:32] "GET /metrics HTTP/1.1" 200 38632 "" "Prometheus/2.43.0" 2024-03-07T03:14:32.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:32 smithi145 bash[21339]: cluster 2024-03-07T03:14:31.531697+0000 mon.a (mon.0) 1239 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2024-03-07T03:14:32.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:32 smithi145 bash[21339]: cluster 2024-03-07T03:14:31.682068+0000 mgr.y (mgr.15132) 35 : cluster 0 pgmap v51: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 277 KiB/s rd, 25 KiB/s wr, 373 op/s 2024-03-07T03:14:32.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:32 smithi145 bash[30906]: cluster 2024-03-07T03:14:31.531697+0000 mon.a (mon.0) 1239 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2024-03-07T03:14:32.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:32 smithi145 bash[30906]: cluster 2024-03-07T03:14:31.682068+0000 mgr.y (mgr.15132) 35 : cluster 0 pgmap v51: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 277 KiB/s rd, 25 KiB/s wr, 373 op/s 2024-03-07T03:14:33.043 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:32 smithi189 bash[60207]: debug 2024-03-07T03:14:32.769+0000 7fecafc80200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:14:33.330 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:33 smithi189 bash[60207]: debug 2024-03-07T03:14:33.049+0000 7fecafc80200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:14:33.330 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:33 smithi189 bash[60207]: debug 2024-03-07T03:14:33.173+0000 7fecafc80200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:14:33.617 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.SnapTrimBeforePrepare (7895 ms) 2024-03-07T03:14:33.618 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.AbortInUseImage 2024-03-07T03:14:33.662 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:14:33.678 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:14:33.761 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:33 smithi189 bash[60207]: debug 2024-03-07T03:14:33.489+0000 7fecafc80200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:14:33.761 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:33 smithi189 bash[59814]: ts=2024-03-07T03:14:33.504Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:14:33.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:33 smithi189 bash[23614]: cluster 2024-03-07T03:14:32.552948+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2024-03-07T03:14:33.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:33 smithi189 bash[23614]: cluster 2024-03-07T03:14:33.477454+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2024-03-07T03:14:33.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:33 smithi189 bash[23614]: audit 2024-03-07T03:14:33.528017+0000 mon.a (mon.0) 1242 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:33.761 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:33 smithi189 bash[23614]: audit 2024-03-07T03:14:33.534242+0000 mon.a (mon.0) 1243 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:33.807 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:33 smithi145 bash[21339]: cluster 2024-03-07T03:14:32.552948+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2024-03-07T03:14:33.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:33 smithi145 bash[21339]: cluster 2024-03-07T03:14:33.477454+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2024-03-07T03:14:33.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:33 smithi145 bash[21339]: audit 2024-03-07T03:14:33.528017+0000 mon.a (mon.0) 1242 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:33.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:33 smithi145 bash[21339]: audit 2024-03-07T03:14:33.534242+0000 mon.a (mon.0) 1243 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:33.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:33 smithi145 bash[30906]: cluster 2024-03-07T03:14:32.552948+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2024-03-07T03:14:33.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:33 smithi145 bash[30906]: cluster 2024-03-07T03:14:33.477454+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2024-03-07T03:14:33.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:33 smithi145 bash[30906]: audit 2024-03-07T03:14:33.528017+0000 mon.a (mon.0) 1242 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:33.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:33 smithi145 bash[30906]: audit 2024-03-07T03:14:33.534242+0000 mon.a (mon.0) 1243 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:33.948 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.AbortInUseImage (331 ms) 2024-03-07T03:14:33.949 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.AbortWithoutSnapshots 2024-03-07T03:14:34.011 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:14:34.035 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:14:34.080 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:33 smithi189 bash[60207]: debug 2024-03-07T03:14:33.765+0000 7fecafc80200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:14:34.118 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:14:34.196 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:14:34.539 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:34 smithi189 bash[60207]: debug 2024-03-07T03:14:34.237+0000 7fecafc80200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:14:34.540 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:34 smithi189 bash[60207]: debug 2024-03-07T03:14:34.377+0000 7fecafc80200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:14:34.602 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:14:34.670 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:14:34.766 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:14:34.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:34 smithi145 bash[21339]: cluster 2024-03-07T03:14:33.682760+0000 mgr.y (mgr.15132) 36 : cluster 0 pgmap v54: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:34.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:34 smithi145 bash[21339]: audit 2024-03-07T03:14:33.784670+0000 mon.c (mon.2) 176 : audit 0 from='client.? 172.21.15.145:0/3989896515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:34.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:34 smithi145 bash[30906]: cluster 2024-03-07T03:14:33.682760+0000 mgr.y (mgr.15132) 36 : cluster 0 pgmap v54: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:34.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:34 smithi145 bash[30906]: audit 2024-03-07T03:14:33.784670+0000 mon.c (mon.2) 176 : audit 0 from='client.? 172.21.15.145:0/3989896515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:34.830 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:34 smithi189 bash[60207]: debug 2024-03-07T03:14:34.669+0000 7fecafc80200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:14:34.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:34 smithi189 bash[23614]: cluster 2024-03-07T03:14:33.682760+0000 mgr.y (mgr.15132) 36 : cluster 0 pgmap v54: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:34.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:34 smithi189 bash[23614]: audit 2024-03-07T03:14:33.784670+0000 mon.c (mon.2) 176 : audit 0 from='client.? 172.21.15.145:0/3989896515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:35.093 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.AbortWithoutSnapshots (1145 ms) 2024-03-07T03:14:35.093 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.AbortWithSnapshots 2024-03-07T03:14:35.162 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:14:35.185 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:14:35.278 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:14:36.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:35 smithi145 bash[21339]: audit 2024-03-07T03:14:34.690165+0000 mon.a (mon.0) 1244 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:36.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:35 smithi145 bash[21339]: audit 2024-03-07T03:14:34.704534+0000 mon.a (mon.0) 1245 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:36.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:35 smithi145 bash[30906]: audit 2024-03-07T03:14:34.690165+0000 mon.a (mon.0) 1244 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:36.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:35 smithi145 bash[30906]: audit 2024-03-07T03:14:34.704534+0000 mon.a (mon.0) 1245 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:36.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:35 smithi189 bash[23614]: audit 2024-03-07T03:14:34.690165+0000 mon.a (mon.0) 1244 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:36.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:35 smithi189 bash[23614]: audit 2024-03-07T03:14:34.704534+0000 mon.a (mon.0) 1245 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:36.081 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:35 smithi189 bash[60207]: debug 2024-03-07T03:14:35.797+0000 7fecafc80200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:14:36.081 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:35 smithi189 bash[60207]: debug 2024-03-07T03:14:35.917+0000 7fecafc80200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:14:36.580 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:36 smithi189 bash[60207]: debug 2024-03-07T03:14:36.221+0000 7fecafc80200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:14:36.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:36 smithi189 bash[60207]: debug 2024-03-07T03:14:36.337+0000 7fecafc80200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:14:36.723 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 8388609~8388609 2024-03-07T03:14:36.757 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 0~1000 2024-03-07T03:14:36.757 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 1010~1000 2024-03-07T03:14:36.757 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:14:36.961 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:36 smithi189 bash[60207]: debug 2024-03-07T03:14:36.817+0000 7fecafc80200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:14:36.961 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:36 smithi189 bash[60207]: debug 2024-03-07T03:14:36.941+0000 7fecafc80200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:14:36.961 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:36 smithi189 bash[23614]: cluster 2024-03-07T03:14:35.684471+0000 mgr.y (mgr.15132) 37 : cluster 0 pgmap v55: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 160 KiB/s rd, 14 KiB/s wr, 217 op/s 2024-03-07T03:14:36.961 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:36 smithi189 bash[23614]: cluster 2024-03-07T03:14:35.711856+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2024-03-07T03:14:36.961 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:36 smithi189 bash[23614]: audit 2024-03-07T03:14:36.234561+0000 mon.a (mon.0) 1247 : audit 0 from='client.? 172.21.15.145:0/2515307565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:37.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:36 smithi145 bash[21339]: cluster 2024-03-07T03:14:35.684471+0000 mgr.y (mgr.15132) 37 : cluster 0 pgmap v55: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 160 KiB/s rd, 14 KiB/s wr, 217 op/s 2024-03-07T03:14:37.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:36 smithi145 bash[21339]: cluster 2024-03-07T03:14:35.711856+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2024-03-07T03:14:37.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:36 smithi145 bash[21339]: audit 2024-03-07T03:14:36.234561+0000 mon.a (mon.0) 1247 : audit 0 from='client.? 172.21.15.145:0/2515307565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:37.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:36 smithi145 bash[30906]: cluster 2024-03-07T03:14:35.684471+0000 mgr.y (mgr.15132) 37 : cluster 0 pgmap v55: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 160 KiB/s rd, 14 KiB/s wr, 217 op/s 2024-03-07T03:14:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:36 smithi145 bash[30906]: cluster 2024-03-07T03:14:35.711856+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2024-03-07T03:14:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:36 smithi145 bash[30906]: audit 2024-03-07T03:14:36.234561+0000 mon.a (mon.0) 1247 : audit 0 from='client.? 172.21.15.145:0/2515307565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:37.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:36 smithi189 bash[60207]: [07/Mar/2024:03:14:36] ENGINE Bus STARTING 2024-03-07T03:14:37.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:37 smithi189 bash[60207]: CherryPy Checker: 2024-03-07T03:14:37.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:37 smithi189 bash[60207]: The Application mounted at '' has an empty config. 2024-03-07T03:14:37.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:37 smithi189 bash[60207]: [07/Mar/2024:03:14:37] ENGINE Serving on http://:::9283 2024-03-07T03:14:37.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:37 smithi189 bash[60207]: [07/Mar/2024:03:14:37] ENGINE Bus STARTED 2024-03-07T03:14:38.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[21339]: cluster 2024-03-07T03:14:36.714112+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2024-03-07T03:14:38.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[21339]: audit 2024-03-07T03:14:36.970882+0000 mon.b (mon.1) 85 : audit 0 from='mgr.? 172.21.15.189:0/328681610' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:14:38.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[21339]: cluster 2024-03-07T03:14:36.971346+0000 mon.a (mon.0) 1249 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:14:38.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[21339]: cluster 2024-03-07T03:14:36.971548+0000 mon.a (mon.0) 1250 : cluster 0 Standby manager daemon x started 2024-03-07T03:14:38.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[21339]: audit 2024-03-07T03:14:36.972074+0000 mon.b (mon.1) 86 : audit 0 from='mgr.? 172.21.15.189:0/328681610' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:14:38.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[21339]: audit 2024-03-07T03:14:36.975006+0000 mon.b (mon.1) 87 : audit 0 from='mgr.? 172.21.15.189:0/328681610' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:14:38.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[21339]: audit 2024-03-07T03:14:36.975487+0000 mon.b (mon.1) 88 : audit 0 from='mgr.? 172.21.15.189:0/328681610' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:14:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[30906]: cluster 2024-03-07T03:14:36.714112+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2024-03-07T03:14:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[30906]: audit 2024-03-07T03:14:36.970882+0000 mon.b (mon.1) 85 : audit 0 from='mgr.? 172.21.15.189:0/328681610' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:14:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[30906]: cluster 2024-03-07T03:14:36.971346+0000 mon.a (mon.0) 1249 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:14:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[30906]: cluster 2024-03-07T03:14:36.971548+0000 mon.a (mon.0) 1250 : cluster 0 Standby manager daemon x started 2024-03-07T03:14:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[30906]: audit 2024-03-07T03:14:36.972074+0000 mon.b (mon.1) 86 : audit 0 from='mgr.? 172.21.15.189:0/328681610' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:14:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[30906]: audit 2024-03-07T03:14:36.975006+0000 mon.b (mon.1) 87 : audit 0 from='mgr.? 172.21.15.189:0/328681610' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:14:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:37 smithi145 bash[30906]: audit 2024-03-07T03:14:36.975487+0000 mon.b (mon.1) 88 : audit 0 from='mgr.? 172.21.15.189:0/328681610' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:14:38.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:37 smithi189 bash[23614]: cluster 2024-03-07T03:14:36.714112+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2024-03-07T03:14:38.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:37 smithi189 bash[23614]: audit 2024-03-07T03:14:36.970882+0000 mon.b (mon.1) 85 : audit 0 from='mgr.? 172.21.15.189:0/328681610' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:14:38.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:37 smithi189 bash[23614]: cluster 2024-03-07T03:14:36.971346+0000 mon.a (mon.0) 1249 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:14:38.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:37 smithi189 bash[23614]: cluster 2024-03-07T03:14:36.971548+0000 mon.a (mon.0) 1250 : cluster 0 Standby manager daemon x started 2024-03-07T03:14:38.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:37 smithi189 bash[23614]: audit 2024-03-07T03:14:36.972074+0000 mon.b (mon.1) 86 : audit 0 from='mgr.? 172.21.15.189:0/328681610' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:14:38.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:37 smithi189 bash[23614]: audit 2024-03-07T03:14:36.975006+0000 mon.b (mon.1) 87 : audit 0 from='mgr.? 172.21.15.189:0/328681610' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:14:38.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:37 smithi189 bash[23614]: audit 2024-03-07T03:14:36.975487+0000 mon.b (mon.1) 88 : audit 0 from='mgr.? 172.21.15.189:0/328681610' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:14:38.503 INFO:tasks.workunit.client.0.smithi145.stdout:write: X 8388609~1000 2024-03-07T03:14:38.520 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 16777228~1000 2024-03-07T03:14:38.521 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 167772160 2024-03-07T03:14:38.541 INFO:tasks.workunit.client.0.smithi145.stdout:write: 2 83886079~8388609 2024-03-07T03:14:38.582 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:14:39.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:38 smithi145 bash[21339]: cluster 2024-03-07T03:14:37.685416+0000 mgr.y (mgr.15132) 38 : cluster 0 pgmap v58: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 14 KiB/s wr, 217 op/s 2024-03-07T03:14:39.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:38 smithi145 bash[21339]: cluster 2024-03-07T03:14:37.752095+0000 mon.a (mon.0) 1251 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2024-03-07T03:14:39.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:38 smithi145 bash[21339]: cluster 2024-03-07T03:14:37.753747+0000 mon.a (mon.0) 1252 : cluster 0 mgrmap e31: y(active, since 40s), standbys: x 2024-03-07T03:14:39.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:38 smithi145 bash[21339]: cluster 2024-03-07T03:14:38.484088+0000 mon.a (mon.0) 1253 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2024-03-07T03:14:39.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:38 smithi145 bash[21339]: audit 2024-03-07T03:14:38.702175+0000 mon.a (mon.0) 1254 : audit 0 from='client.? 172.21.15.145:0/480323227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:39.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:38 smithi145 bash[30906]: cluster 2024-03-07T03:14:37.685416+0000 mgr.y (mgr.15132) 38 : cluster 0 pgmap v58: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 14 KiB/s wr, 217 op/s 2024-03-07T03:14:39.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:38 smithi145 bash[30906]: cluster 2024-03-07T03:14:37.752095+0000 mon.a (mon.0) 1251 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2024-03-07T03:14:39.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:38 smithi145 bash[30906]: cluster 2024-03-07T03:14:37.753747+0000 mon.a (mon.0) 1252 : cluster 0 mgrmap e31: y(active, since 40s), standbys: x 2024-03-07T03:14:39.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:38 smithi145 bash[30906]: cluster 2024-03-07T03:14:38.484088+0000 mon.a (mon.0) 1253 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2024-03-07T03:14:39.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:38 smithi145 bash[30906]: audit 2024-03-07T03:14:38.702175+0000 mon.a (mon.0) 1254 : audit 0 from='client.? 172.21.15.145:0/480323227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:39.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:38 smithi189 bash[23614]: cluster 2024-03-07T03:14:37.685416+0000 mgr.y (mgr.15132) 38 : cluster 0 pgmap v58: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 161 KiB/s rd, 14 KiB/s wr, 217 op/s 2024-03-07T03:14:39.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:38 smithi189 bash[23614]: cluster 2024-03-07T03:14:37.752095+0000 mon.a (mon.0) 1251 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2024-03-07T03:14:39.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:38 smithi189 bash[23614]: cluster 2024-03-07T03:14:37.753747+0000 mon.a (mon.0) 1252 : cluster 0 mgrmap e31: y(active, since 40s), standbys: x 2024-03-07T03:14:39.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:38 smithi189 bash[23614]: cluster 2024-03-07T03:14:38.484088+0000 mon.a (mon.0) 1253 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2024-03-07T03:14:39.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:38 smithi189 bash[23614]: audit 2024-03-07T03:14:38.702175+0000 mon.a (mon.0) 1254 : audit 0 from='client.? 172.21.15.145:0/480323227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:40.516 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:14:40.569 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 83886079~1 2024-03-07T03:14:40.590 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:14:40.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:40 smithi189 bash[23614]: cluster 2024-03-07T03:14:39.498324+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2024-03-07T03:14:40.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:40 smithi189 bash[23614]: cluster 2024-03-07T03:14:39.686298+0000 mgr.y (mgr.15132) 39 : cluster 0 pgmap v62: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 33 MiB/s wr, 465 op/s 2024-03-07T03:14:40.854 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:40 smithi145 bash[21339]: cluster 2024-03-07T03:14:39.498324+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2024-03-07T03:14:40.854 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:40 smithi145 bash[21339]: cluster 2024-03-07T03:14:39.686298+0000 mgr.y (mgr.15132) 39 : cluster 0 pgmap v62: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 33 MiB/s wr, 465 op/s 2024-03-07T03:14:40.854 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:40 smithi145 bash[30906]: cluster 2024-03-07T03:14:39.498324+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2024-03-07T03:14:40.854 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:40 smithi145 bash[30906]: cluster 2024-03-07T03:14:39.686298+0000 mgr.y (mgr.15132) 39 : cluster 0 pgmap v62: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 33 MiB/s wr, 465 op/s 2024-03-07T03:14:41.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:41 smithi145 bash[21339]: cluster 2024-03-07T03:14:40.505803+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2024-03-07T03:14:41.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:41 smithi145 bash[21339]: audit 2024-03-07T03:14:41.146378+0000 mon.a (mon.0) 1257 : audit 0 from='client.? 172.21.15.145:0/1980152202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:41.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:41 smithi145 bash[30906]: cluster 2024-03-07T03:14:40.505803+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2024-03-07T03:14:41.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:41 smithi145 bash[30906]: audit 2024-03-07T03:14:41.146378+0000 mon.a (mon.0) 1257 : audit 0 from='client.? 172.21.15.145:0/1980152202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:41.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:41 smithi189 bash[23614]: cluster 2024-03-07T03:14:40.505803+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2024-03-07T03:14:41.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:41 smithi189 bash[23614]: audit 2024-03-07T03:14:41.146378+0000 mon.a (mon.0) 1257 : audit 0 from='client.? 172.21.15.145:0/1980152202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:42.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:42 smithi145 bash[21339]: cluster 2024-03-07T03:14:41.506890+0000 mon.a (mon.0) 1258 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2024-03-07T03:14:42.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:42 smithi145 bash[21339]: cluster 2024-03-07T03:14:41.687036+0000 mgr.y (mgr.15132) 40 : cluster 0 pgmap v65: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 33 MiB/s wr, 468 op/s 2024-03-07T03:14:42.808 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:14:42] "GET /metrics HTTP/1.1" 200 38632 "" "Prometheus/2.43.0" 2024-03-07T03:14:42.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:42 smithi145 bash[30906]: cluster 2024-03-07T03:14:41.506890+0000 mon.a (mon.0) 1258 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2024-03-07T03:14:42.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:42 smithi145 bash[30906]: cluster 2024-03-07T03:14:41.687036+0000 mgr.y (mgr.15132) 40 : cluster 0 pgmap v65: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 33 MiB/s wr, 468 op/s 2024-03-07T03:14:42.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:42 smithi189 bash[23614]: cluster 2024-03-07T03:14:41.506890+0000 mon.a (mon.0) 1258 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2024-03-07T03:14:42.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:42 smithi189 bash[23614]: cluster 2024-03-07T03:14:41.687036+0000 mgr.y (mgr.15132) 40 : cluster 0 pgmap v65: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 33 MiB/s wr, 468 op/s 2024-03-07T03:14:43.806 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:43 smithi189 bash[23614]: cluster 2024-03-07T03:14:42.526767+0000 mon.a (mon.0) 1259 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2024-03-07T03:14:43.806 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:43 smithi189 bash[23614]: audit 2024-03-07T03:14:42.806366+0000 mon.a (mon.0) 1260 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:43.807 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:43 smithi189 bash[59814]: ts=2024-03-07T03:14:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:14:43.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:43 smithi145 bash[21339]: cluster 2024-03-07T03:14:42.526767+0000 mon.a (mon.0) 1259 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2024-03-07T03:14:43.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:43 smithi145 bash[21339]: audit 2024-03-07T03:14:42.806366+0000 mon.a (mon.0) 1260 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:43.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:43 smithi145 bash[30906]: cluster 2024-03-07T03:14:42.526767+0000 mon.a (mon.0) 1259 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2024-03-07T03:14:43.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:43 smithi145 bash[30906]: audit 2024-03-07T03:14:42.806366+0000 mon.a (mon.0) 1260 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:44.332 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:14:44.477 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:14:44.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: cluster 2024-03-07T03:14:43.534597+0000 mon.a (mon.0) 1261 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2024-03-07T03:14:44.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: audit 2024-03-07T03:14:43.599543+0000 mon.a (mon.0) 1262 : audit 0 from='client.? 172.21.15.145:0/1705385601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:44.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: cluster 2024-03-07T03:14:43.687500+0000 mgr.y (mgr.15132) 41 : cluster 0 pgmap v68: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: audit 2024-03-07T03:14:44.371595+0000 mon.a (mon.0) 1263 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: audit 2024-03-07T03:14:44.387962+0000 mon.a (mon.0) 1264 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: audit 2024-03-07T03:14:44.391788+0000 mon.a (mon.0) 1265 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: audit 2024-03-07T03:14:44.393819+0000 mon.a (mon.0) 1266 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:14:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: audit 2024-03-07T03:14:44.413002+0000 mon.a (mon.0) 1267 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: audit 2024-03-07T03:14:44.504334+0000 mon.a (mon.0) 1268 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: audit 2024-03-07T03:14:44.507738+0000 mon.a (mon.0) 1269 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:14:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: audit 2024-03-07T03:14:44.518333+0000 mon.a (mon.0) 1270 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: audit 2024-03-07T03:14:44.519751+0000 mon.a (mon.0) 1271 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-07T03:14:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: audit 2024-03-07T03:14:44.534012+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-07T03:14:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[21339]: audit 2024-03-07T03:14:44.535282+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-07T03:14:44.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: cluster 2024-03-07T03:14:43.534597+0000 mon.a (mon.0) 1261 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2024-03-07T03:14:44.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: audit 2024-03-07T03:14:43.599543+0000 mon.a (mon.0) 1262 : audit 0 from='client.? 172.21.15.145:0/1705385601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:44.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: cluster 2024-03-07T03:14:43.687500+0000 mgr.y (mgr.15132) 41 : cluster 0 pgmap v68: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:44.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: audit 2024-03-07T03:14:44.371595+0000 mon.a (mon.0) 1263 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:44.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: audit 2024-03-07T03:14:44.387962+0000 mon.a (mon.0) 1264 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:44.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: audit 2024-03-07T03:14:44.391788+0000 mon.a (mon.0) 1265 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:44.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: audit 2024-03-07T03:14:44.393819+0000 mon.a (mon.0) 1266 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:14:44.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: audit 2024-03-07T03:14:44.413002+0000 mon.a (mon.0) 1267 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:44.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: audit 2024-03-07T03:14:44.504334+0000 mon.a (mon.0) 1268 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:44.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: audit 2024-03-07T03:14:44.507738+0000 mon.a (mon.0) 1269 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:14:44.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: audit 2024-03-07T03:14:44.518333+0000 mon.a (mon.0) 1270 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:44.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: audit 2024-03-07T03:14:44.519751+0000 mon.a (mon.0) 1271 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-07T03:14:44.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: audit 2024-03-07T03:14:44.534012+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-07T03:14:44.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:44 smithi145 bash[30906]: audit 2024-03-07T03:14:44.535282+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-07T03:14:44.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: cluster 2024-03-07T03:14:43.534597+0000 mon.a (mon.0) 1261 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2024-03-07T03:14:44.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: audit 2024-03-07T03:14:43.599543+0000 mon.a (mon.0) 1262 : audit 0 from='client.? 172.21.15.145:0/1705385601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:44.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: cluster 2024-03-07T03:14:43.687500+0000 mgr.y (mgr.15132) 41 : cluster 0 pgmap v68: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:14:44.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: audit 2024-03-07T03:14:44.371595+0000 mon.a (mon.0) 1263 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:44.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: audit 2024-03-07T03:14:44.387962+0000 mon.a (mon.0) 1264 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:44.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: audit 2024-03-07T03:14:44.391788+0000 mon.a (mon.0) 1265 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:44.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: audit 2024-03-07T03:14:44.393819+0000 mon.a (mon.0) 1266 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:14:44.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: audit 2024-03-07T03:14:44.413002+0000 mon.a (mon.0) 1267 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:44.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: audit 2024-03-07T03:14:44.504334+0000 mon.a (mon.0) 1268 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:44.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: audit 2024-03-07T03:14:44.507738+0000 mon.a (mon.0) 1269 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:14:44.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: audit 2024-03-07T03:14:44.518333+0000 mon.a (mon.0) 1270 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:44.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: audit 2024-03-07T03:14:44.519751+0000 mon.a (mon.0) 1271 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-07T03:14:44.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: audit 2024-03-07T03:14:44.534012+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-07T03:14:44.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:44 smithi189 bash[23614]: audit 2024-03-07T03:14:44.535282+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-07T03:14:45.591 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 0~8388609 2024-03-07T03:14:45.632 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 16777218~8388609 2024-03-07T03:14:45.739 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:14:45.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:45 smithi189 bash[23614]: cephadm 2024-03-07T03:14:44.509202+0000 mgr.y (mgr.15132) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2024-03-07T03:14:45.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:45 smithi189 bash[23614]: audit 2024-03-07T03:14:44.552324+0000 mon.a (mon.0) 1274 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-07T03:14:45.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:45 smithi189 bash[23614]: audit 2024-03-07T03:14:44.565450+0000 mon.a (mon.0) 1275 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:14:45.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:45 smithi189 bash[23614]: audit 2024-03-07T03:14:44.566441+0000 mon.a (mon.0) 1276 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-07T03:14:45.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:45 smithi189 bash[23614]: cephadm 2024-03-07T03:14:44.567002+0000 mgr.y (mgr.15132) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2024-03-07T03:14:46.006 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:45 smithi145 bash[21339]: cephadm 2024-03-07T03:14:44.509202+0000 mgr.y (mgr.15132) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2024-03-07T03:14:46.006 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:45 smithi145 bash[21339]: audit 2024-03-07T03:14:44.552324+0000 mon.a (mon.0) 1274 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-07T03:14:46.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:45 smithi145 bash[21339]: audit 2024-03-07T03:14:44.565450+0000 mon.a (mon.0) 1275 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:14:46.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:45 smithi145 bash[21339]: audit 2024-03-07T03:14:44.566441+0000 mon.a (mon.0) 1276 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-07T03:14:46.007 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:45 smithi145 bash[21339]: cephadm 2024-03-07T03:14:44.567002+0000 mgr.y (mgr.15132) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2024-03-07T03:14:46.007 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:45 smithi145 bash[30906]: cephadm 2024-03-07T03:14:44.509202+0000 mgr.y (mgr.15132) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2024-03-07T03:14:46.007 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:45 smithi145 bash[30906]: audit 2024-03-07T03:14:44.552324+0000 mon.a (mon.0) 1274 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-07T03:14:46.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:45 smithi145 bash[30906]: audit 2024-03-07T03:14:44.565450+0000 mon.a (mon.0) 1275 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:14:46.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:45 smithi145 bash[30906]: audit 2024-03-07T03:14:44.566441+0000 mon.a (mon.0) 1276 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-07T03:14:46.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:45 smithi145 bash[30906]: cephadm 2024-03-07T03:14:44.567002+0000 mgr.y (mgr.15132) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2024-03-07T03:14:46.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:46 smithi145 bash[21339]: cluster 2024-03-07T03:14:45.571283+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2024-03-07T03:14:46.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:46 smithi145 bash[21339]: cluster 2024-03-07T03:14:45.688231+0000 mgr.y (mgr.15132) 44 : cluster 0 pgmap v70: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 11 KiB/s wr, 183 op/s 2024-03-07T03:14:46.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:46 smithi145 bash[21339]: audit 2024-03-07T03:14:46.095887+0000 mon.c (mon.2) 177 : audit 0 from='client.? 172.21.15.145:0/3927106289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:46.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:46 smithi145 bash[30906]: cluster 2024-03-07T03:14:45.571283+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2024-03-07T03:14:46.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:46 smithi145 bash[30906]: cluster 2024-03-07T03:14:45.688231+0000 mgr.y (mgr.15132) 44 : cluster 0 pgmap v70: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 11 KiB/s wr, 183 op/s 2024-03-07T03:14:46.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:46 smithi145 bash[30906]: audit 2024-03-07T03:14:46.095887+0000 mon.c (mon.2) 177 : audit 0 from='client.? 172.21.15.145:0/3927106289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:46.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:46 smithi189 bash[23614]: cluster 2024-03-07T03:14:45.571283+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2024-03-07T03:14:46.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:46 smithi189 bash[23614]: cluster 2024-03-07T03:14:45.688231+0000 mgr.y (mgr.15132) 44 : cluster 0 pgmap v70: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 11 KiB/s wr, 183 op/s 2024-03-07T03:14:46.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:46 smithi189 bash[23614]: audit 2024-03-07T03:14:46.095887+0000 mon.c (mon.2) 177 : audit 0 from='client.? 172.21.15.145:0/3927106289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:47.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[21339]: cluster 2024-03-07T03:14:46.586316+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2024-03-07T03:14:47.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[21339]: cephadm 2024-03-07T03:14:46.986509+0000 mgr.y (mgr.15132) 45 : cephadm 1 Upgrade: Updating mon.a 2024-03-07T03:14:47.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[21339]: audit 2024-03-07T03:14:46.995828+0000 mon.a (mon.0) 1279 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:47.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[21339]: audit 2024-03-07T03:14:46.997158+0000 mon.a (mon.0) 1280 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:14:47.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[21339]: audit 2024-03-07T03:14:46.998279+0000 mon.a (mon.0) 1281 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:14:47.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[21339]: audit 2024-03-07T03:14:46.999495+0000 mon.a (mon.0) 1282 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:47.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[21339]: cephadm 2024-03-07T03:14:47.000917+0000 mgr.y (mgr.15132) 46 : cephadm 1 Deploying daemon mon.a on smithi145 2024-03-07T03:14:47.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[30906]: cluster 2024-03-07T03:14:46.586316+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2024-03-07T03:14:47.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[30906]: cephadm 2024-03-07T03:14:46.986509+0000 mgr.y (mgr.15132) 45 : cephadm 1 Upgrade: Updating mon.a 2024-03-07T03:14:47.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[30906]: audit 2024-03-07T03:14:46.995828+0000 mon.a (mon.0) 1279 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:47.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[30906]: audit 2024-03-07T03:14:46.997158+0000 mon.a (mon.0) 1280 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:14:47.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[30906]: audit 2024-03-07T03:14:46.998279+0000 mon.a (mon.0) 1281 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:14:47.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[30906]: audit 2024-03-07T03:14:46.999495+0000 mon.a (mon.0) 1282 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:47.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:47 smithi145 bash[30906]: cephadm 2024-03-07T03:14:47.000917+0000 mgr.y (mgr.15132) 46 : cephadm 1 Deploying daemon mon.a on smithi145 2024-03-07T03:14:48.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:47 smithi189 bash[23614]: cluster 2024-03-07T03:14:46.586316+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2024-03-07T03:14:48.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:47 smithi189 bash[23614]: cephadm 2024-03-07T03:14:46.986509+0000 mgr.y (mgr.15132) 45 : cephadm 1 Upgrade: Updating mon.a 2024-03-07T03:14:48.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:47 smithi189 bash[23614]: audit 2024-03-07T03:14:46.995828+0000 mon.a (mon.0) 1279 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' 2024-03-07T03:14:48.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:47 smithi189 bash[23614]: audit 2024-03-07T03:14:46.997158+0000 mon.a (mon.0) 1280 : audit 1 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:14:48.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:47 smithi189 bash[23614]: audit 2024-03-07T03:14:46.998279+0000 mon.a (mon.0) 1281 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:14:48.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:47 smithi189 bash[23614]: audit 2024-03-07T03:14:46.999495+0000 mon.a (mon.0) 1282 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:48.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:47 smithi189 bash[23614]: cephadm 2024-03-07T03:14:47.000917+0000 mgr.y (mgr.15132) 46 : cephadm 1 Deploying daemon mon.a on smithi145 2024-03-07T03:14:48.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:48 smithi145 bash[21339]: cluster 2024-03-07T03:14:47.598172+0000 mon.a (mon.0) 1283 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2024-03-07T03:14:48.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:48 smithi145 bash[21339]: cluster 2024-03-07T03:14:47.688813+0000 mgr.y (mgr.15132) 47 : cluster 0 pgmap v73: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 11 KiB/s wr, 184 op/s 2024-03-07T03:14:48.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:48 smithi145 bash[21339]: audit 2024-03-07T03:14:48.540075+0000 mon.a (mon.0) 1284 : audit 0 from='client.? 172.21.15.145:0/4155066133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:48.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:48 smithi145 bash[30906]: cluster 2024-03-07T03:14:47.598172+0000 mon.a (mon.0) 1283 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2024-03-07T03:14:48.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:48 smithi145 bash[30906]: cluster 2024-03-07T03:14:47.688813+0000 mgr.y (mgr.15132) 47 : cluster 0 pgmap v73: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 11 KiB/s wr, 184 op/s 2024-03-07T03:14:48.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:48 smithi145 bash[30906]: audit 2024-03-07T03:14:48.540075+0000 mon.a (mon.0) 1284 : audit 0 from='client.? 172.21.15.145:0/4155066133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:48 smithi189 bash[23614]: cluster 2024-03-07T03:14:47.598172+0000 mon.a (mon.0) 1283 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2024-03-07T03:14:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:48 smithi189 bash[23614]: cluster 2024-03-07T03:14:47.688813+0000 mgr.y (mgr.15132) 47 : cluster 0 pgmap v73: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 11 KiB/s wr, 184 op/s 2024-03-07T03:14:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:48 smithi189 bash[23614]: audit 2024-03-07T03:14:48.540075+0000 mon.a (mon.0) 1284 : audit 0 from='client.? 172.21.15.145:0/4155066133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:49.774 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:14:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:49.775 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:14:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:49.775 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:49 smithi145 bash[21339]: cluster 2024-03-07T03:14:48.613566+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2024-03-07T03:14:49.775 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:49.775 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:14:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:49.776 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:14:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:49.776 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:14:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:49.776 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:49.777 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:14:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:49.777 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:49 smithi145 bash[30906]: cluster 2024-03-07T03:14:48.613566+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2024-03-07T03:14:49.777 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:49 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:50.042 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:49 smithi145 systemd[1]: Stopping Ceph mon.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:14:50.042 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:49 smithi145 bash[21339]: debug 2024-03-07T03:14:49.927+0000 7f2338844700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T03:14:50.042 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:49 smithi145 bash[21339]: debug 2024-03-07T03:14:49.927+0000 7f2338844700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-03-07T03:14:50.042 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:49 smithi145 bash[72238]: [07/Mar/2024:03:14:49] ENGINE Bus STOPPING 2024-03-07T03:14:50.042 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:49 smithi145 bash[72238]: [07/Mar/2024:03:14:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:14:50.042 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:49 smithi145 bash[72238]: [07/Mar/2024:03:14:49] ENGINE Bus STOPPED 2024-03-07T03:14:50.042 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:49 smithi145 bash[72238]: [07/Mar/2024:03:14:49] ENGINE Bus STARTING 2024-03-07T03:14:50.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:49 smithi189 bash[23614]: cluster 2024-03-07T03:14:48.613566+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2024-03-07T03:14:50.097 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.AbortWithSnapshots (15004 ms) 2024-03-07T03:14:50.097 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.CloneV1Parent 2024-03-07T03:14:50.142 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:14:50.166 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 0~10 2024-03-07T03:14:50.185 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:14:50.308 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:50 smithi145 bash[72238]: [07/Mar/2024:03:14:50] ENGINE Serving on http://:::9283 2024-03-07T03:14:50.308 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:50 smithi145 bash[72238]: [07/Mar/2024:03:14:50] ENGINE Bus STARTED 2024-03-07T03:14:50.997 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:50 smithi145 bash[79726]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mon-a 2024-03-07T03:14:50.997 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:50 smithi145 bash[79820]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mon-a 2024-03-07T03:14:50.997 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:50 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.a.service: Deactivated successfully. 2024-03-07T03:14:50.998 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:50 smithi145 systemd[1]: Stopped Ceph mon.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:14:51.308 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:14:51 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:51.308 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:14:51 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:51.309 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:14:51 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:51.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:51 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:51.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:51 smithi145 systemd[1]: Started Ceph mon.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:14:51.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:51 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:51.310 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:14:51 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:51.310 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:14:51 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:51.310 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:14:51 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:51.310 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:51 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:52.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.195+0000 7fdb55f66c80 0 set uid:gid to 167:167 (ceph:ceph) 2024-03-07T03:14:52.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.195+0000 7fdb55f66c80 0 ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev), process ceph-mon, pid 7 2024-03-07T03:14:52.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.195+0000 7fdb55f66c80 0 pidfile_write: ignore empty --pid-file 2024-03-07T03:14:52.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.199+0000 7fdb55f66c80 0 load: jerasure load: lrc 2024-03-07T03:14:52.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: RocksDB version: 7.9.2 2024-03-07T03:14:52.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Git sha 0 2024-03-07T03:14:52.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Compile date 2024-03-06 19:59:24 2024-03-07T03:14:52.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: DB SUMMARY 2024-03-07T03:14:52.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: DB Session ID: OBK3JLMSBJDBIV8XSIEW 2024-03-07T03:14:52.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: CURRENT file: CURRENT 2024-03-07T03:14:52.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: IDENTITY file: IDENTITY 2024-03-07T03:14:52.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: MANIFEST file: MANIFEST-000015 size: 2492 Bytes 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000044.sst 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000042.log size: 252790 ; 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.error_if_exists: 0 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.create_if_missing: 0 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.paranoid_checks: 1 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.env: 0x55e03cbceb20 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.fs: PosixFileSystem 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.info_log: 0x55e03ee17920 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_file_opening_threads: 16 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.statistics: (nil) 2024-03-07T03:14:52.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.use_fsync: 0 2024-03-07T03:14:52.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_log_file_size: 0 2024-03-07T03:14:52.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-07T03:14:52.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-03-07T03:14:52.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.keep_log_file_num: 1000 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.recycle_log_file_num: 0 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.allow_fallocate: 1 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.allow_mmap_reads: 0 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.allow_mmap_writes: 0 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.use_direct_reads: 0 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.create_missing_column_families: 0 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.db_log_dir: 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.wal_dir: 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-03-07T03:14:52.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.advise_random_on_open: 1 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.db_write_buffer_size: 0 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.write_buffer_manager: 0x55e03edeadc0 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.rate_limiter: (nil) 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.wal_recovery_mode: 2 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.enable_thread_tracking: 0 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.enable_pipelined_write: 0 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.unordered_write: 0 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-07T03:14:52.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.row_cache: None 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.wal_filter: None 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.allow_ingest_behind: 0 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.two_write_queues: 0 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.manual_wal_flush: 0 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.wal_compression: 0 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.atomic_flush: 0 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.log_readahead_size: 0 2024-03-07T03:14:52.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.best_efforts_recovery: 0 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.allow_data_in_errors: 0 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.db_host_id: __hostname__ 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_background_jobs: 2 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_background_compactions: -1 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_subcompactions: 1 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_total_wal_size: 0 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-07T03:14:52.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_open_files: -1 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.bytes_per_sync: 0 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compaction_readahead_size: 0 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_background_flushes: -1 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Compression algorithms supported: 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: kZSTD supported: 0 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: kXpressCompression supported: 0 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: kLZ4HCCompression supported: 1 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: kZlibCompression supported: 1 2024-03-07T03:14:52.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: kSnappyCompression supported: 1 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: kLZ4Compression supported: 1 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: kBZip2Compression supported: 0 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.merge_operator: 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compaction_filter: None 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compaction_filter_factory: None 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.sst_partitioner_factory: None 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e03ee17a20) 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: cache_index_and_filter_blocks: 1 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-07T03:14:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: pin_top_level_index_and_filter: 1 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: index_type: 0 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: data_block_index_type: 0 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: index_shortening: 1 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: data_block_hash_table_util_ratio: 0.750000 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: checksum: 4 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: no_block_cache: 0 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: block_cache: 0x55e03ee1b090 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: block_cache_name: BinnedLRUCache 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: block_cache_options: 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: capacity : 536870912 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: num_shard_bits : 4 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: strict_capacity_limit : 0 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: high_pri_pool_ratio: 0.000 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: block_cache_compressed: (nil) 2024-03-07T03:14:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: persistent_cache: (nil) 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: block_size: 4096 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: block_size_deviation: 10 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: block_restart_interval: 16 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: index_block_restart_interval: 1 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: metadata_block_size: 4096 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: partition_filters: 0 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: use_delta_encoding: 1 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: filter_policy: bloomfilter 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: whole_key_filtering: 1 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: verify_compression: 0 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: read_amp_bytes_per_bit: 0 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: format_version: 5 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: enable_index_compression: 1 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: block_align: 0 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: max_auto_readahead_size: 262144 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: prepopulate_block_cache: 0 2024-03-07T03:14:52.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: initial_auto_readahead_size: 8192 2024-03-07T03:14:52.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: num_file_reads_for_auto_readahead: 2 2024-03-07T03:14:52.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.write_buffer_size: 33554432 2024-03-07T03:14:52.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_write_buffer_number: 2 2024-03-07T03:14:52.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compression: NoCompression 2024-03-07T03:14:52.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.bottommost_compression: Disabled 2024-03-07T03:14:52.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.prefix_extractor: nullptr 2024-03-07T03:14:52.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-07T03:14:52.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.num_levels: 7 2024-03-07T03:14:52.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-07T03:14:52.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-07T03:14:52.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-07T03:14:52.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-07T03:14:52.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compression_opts.level: 32767 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compression_opts.strategy: 0 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:14:52.572 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compression_opts.enabled: false 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.target_file_size_base: 67108864 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-07T03:14:52.573 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.arena_block_size: 1048576 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.disable_auto_compactions: 0 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-07T03:14:52.574 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.table_properties_collectors: 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.inplace_update_support: 0 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.bloom_locality: 0 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.max_successive_merges: 0 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.paranoid_file_checks: 0 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.force_consistency_checks: 1 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.report_bg_io_stats: 0 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.ttl: 2592000 2024-03-07T03:14:52.575 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.enable_blob_files: false 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.min_blob_size: 0 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.blob_file_size: 268435456 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.blob_file_starting_level: 0 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 46, last_sequence is 20904, log_number is 42,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 42 2024-03-07T03:14:52.576 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 42 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: c046e38c-0770-4dfa-8688-b8a45c8198a5 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781292208408, "job": 1, "event": "recovery_started", "wal_files": [42]} 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.203+0000 7fdb55f66c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #42 mode 2 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.207+0000 7fdb55f66c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781292210276, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 47, "file_size": 249516, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 20905, "largest_seqno": 21141, "table_properties": {"data_size": 247519, "index_size": 665, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 3328, "raw_average_key_size": 26, "raw_value_size": 244685, "raw_average_value_size": 1973, "num_data_blocks": 29, "num_entries": 124, "num_filter_entries": 124, "num_deletions": 5, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1709781292, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "c046e38c-0770-4dfa-8688-b8a45c8198a5", "db_session_id": "OBK3JLMSBJDBIV8XSIEW", "orig_file_number": 47, "seqno_to_time_mapping": "N/A"}} 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.207+0000 7fdb55f66c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781292210375, "job": 1, "event": "recovery_finished"} 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.207+0000 7fdb55f66c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 49 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.207+0000 7fdb55f66c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.207+0000 7fdb55f66c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000042.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.207+0000 7fdb55f66c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e03ef12000 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.207+0000 7fdb55f66c80 4 rocksdb: DB pointer 0x55e03eefc000 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.207+0000 7fdb446cf700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.207+0000 7fdb446cf700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: ** DB Stats ** 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-07T03:14:52.577 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: ** Compaction Stats [default] ** 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: L0 1/0 243.67 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 184.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: L6 1/0 9.91 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Sum 2/0 10.15 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 184.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 184.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: ** Compaction Stats [default] ** 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 184.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-07T03:14:52.578 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Flush(GB): cumulative 0.000, interval 0.000 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: AddFile(Total Files): cumulative 0, interval 0 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: AddFile(L0 Files): cumulative 0, interval 0 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: AddFile(Keys): cumulative 0, interval 0 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Cumulative compaction: 0.00 GB write, 38.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Interval compaction: 0.00 GB write, 38.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Block cache BinnedLRUCache@0x55e03ee1b090#7 capacity: 512.00 MB usage: 1.14 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,0.77 KB,0.000146031%) Misc(1,0.00 KB,0%) 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: ** File Read Latency Histogram By Level [default] ** 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.207+0000 7fdb55f66c80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] at bind addrs [v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.211+0000 7fdb55f66c80 1 mon.a@-1(???) e3 preinit fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:14:52.579 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.211+0000 7fdb55f66c80 0 mon.a@-1(???).mds e10 new map 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.211+0000 7fdb55f66c80 0 mon.a@-1(???).mds e10 print_map 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: e10 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: enable_multiple, ever_enabled_multiple: 1,1 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: legacy client fscid: 1 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Filesystem 'foo' (1) 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: fs_name foo 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: epoch 10 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: created 2024-03-07T03:08:49.405527+0000 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: modified 2024-03-07T03:09:00.347215+0000 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: tableserver 0 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: root 0 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: session_timeout 60 2024-03-07T03:14:52.580 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: session_autoclose 300 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: max_file_size 1099511627776 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: max_xattr_size 65536 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: required_client_features {} 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: last_failure 0 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: last_failure_osd_epoch 70 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: max_mds 1 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: in 0 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: up {0=24376} 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: failed 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: damaged 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: stopped 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: data_pools [4] 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: metadata_pool 3 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: inline_data disabled 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: balancer 2024-03-07T03:14:52.581 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: bal_rank_mask -1 2024-03-07T03:14:52.582 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: standby_count_wanted 1 2024-03-07T03:14:52.582 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: [mds.foo.smithi189.zkinbe{0:24376} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] compat {c=[1],r=[1],i=[7ff]}] 2024-03-07T03:14:52.582 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: 2024-03-07T03:14:52.582 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: 2024-03-07T03:14:52.582 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: Standby daemons: 2024-03-07T03:14:52.582 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: 2024-03-07T03:14:52.582 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: [mds.foo.smithi145.zmgvgw{-1:14589} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.145:6834/4165785656,v1:172.21.15.145:6835/4165785656] compat {c=[1],r=[1],i=[7ff]}] 2024-03-07T03:14:52.582 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.211+0000 7fdb55f66c80 0 mon.a@-1(???).osd e271 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-07T03:14:52.582 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.211+0000 7fdb55f66c80 0 mon.a@-1(???).osd e271 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:14:52.582 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.211+0000 7fdb55f66c80 0 mon.a@-1(???).osd e271 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:14:52.582 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.211+0000 7fdb55f66c80 0 mon.a@-1(???).osd e271 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:14:52.582 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[79878]: debug 2024-03-07T03:14:52.211+0000 7fdb55f66c80 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-03-07T03:14:52.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:52 smithi189 bash[59814]: ts=2024-03-07T03:14:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:14:53.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:14:52] "GET /metrics HTTP/1.1" 200 38628 "" "Prometheus/2.43.0" 2024-03-07T03:14:53.749 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[59814]: ts=2024-03-07T03:14:53.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:14:53.749 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: audit 2024-03-07T03:14:49.939432+0000 mon.c (mon.2) 178 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:14:53.749 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: audit 2024-03-07T03:14:49.942759+0000 mon.c (mon.2) 179 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:14:53.749 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: audit 2024-03-07T03:14:49.943066+0000 mon.c (mon.2) 180 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:14:53.749 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: audit 2024-03-07T03:14:50.986262+0000 mon.c (mon.2) 181 : audit 0 from='client.? 172.21.15.145:0/228987950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:53.749 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: cluster 2024-03-07T03:14:51.689979+0000 mgr.y (mgr.15132) 49 : cluster 0 pgmap v77: 137 pgs: 137 active+clean; 937 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 18 MiB/s wr, 246 op/s 2024-03-07T03:14:53.749 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: audit 2024-03-07T03:14:53.424116+0000 mon.c (mon.2) 182 : audit 0 from='client.? 172.21.15.145:0/2943588756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:53.749 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: cluster 2024-03-07T03:14:53.636200+0000 mon.a (mon.0) 1 : cluster 1 mon.a calling monitor election 2024-03-07T03:14:53.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: cluster 2024-03-07T03:14:53.660602+0000 mon.a (mon.0) 2 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:14:53.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: cluster 2024-03-07T03:14:53.671708+0000 mon.a (mon.0) 3 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:14:53.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: cluster 2024-03-07T03:14:53.691224+0000 mon.a (mon.0) 4 : cluster 0 fsmap foo:1 {0=foo.smithi189.zkinbe=up:active} 1 up:standby 2024-03-07T03:14:53.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: cluster 2024-03-07T03:14:53.691321+0000 mon.a (mon.0) 5 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2024-03-07T03:14:53.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: cluster 2024-03-07T03:14:53.693584+0000 mon.a (mon.0) 6 : cluster 0 mgrmap e31: y(active, since 56s), standbys: x 2024-03-07T03:14:53.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: cluster 2024-03-07T03:14:53.694809+0000 mon.a (mon.0) 7 : cluster 3 Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-03-07T03:14:53.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: cluster 2024-03-07T03:14:53.694850+0000 mon.a (mon.0) 8 : cluster 3 [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-07T03:14:53.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: cluster 2024-03-07T03:14:53.694876+0000 mon.a (mon.0) 9 : cluster 3 application not enabled on pool 'test-librbd-smithi145-68337-9' 2024-03-07T03:14:53.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: cluster 2024-03-07T03:14:53.694921+0000 mon.a (mon.0) 10 : cluster 3 use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-07T03:14:53.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: audit 2024-03-07T03:14:53.713351+0000 mon.a (mon.0) 11 : audit 1 from='mgr.15132 ' entity='' 2024-03-07T03:14:53.750 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[23614]: cluster 2024-03-07T03:14:53.714552+0000 mon.a (mon.0) 12 : cluster 0 mgrmap e32: y(active, since 56s), standbys: x 2024-03-07T03:14:54.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[72238]: ignoring --setuser ceph since I am not root 2024-03-07T03:14:54.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[72238]: ignoring --setgroup ceph since I am not root 2024-03-07T03:14:54.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[72238]: debug 2024-03-07T03:14:53.783+0000 7f31c4007700 1 -- 172.21.15.145:0/3221504441 <== mon.2 v2:172.21.15.145:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55589dba45a0 con 0x55589e924400 2024-03-07T03:14:54.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[72238]: debug 2024-03-07T03:14:53.931+0000 7f31cddeb200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:14:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: audit 2024-03-07T03:14:49.939432+0000 mon.c (mon.2) 178 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:14:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: audit 2024-03-07T03:14:49.942759+0000 mon.c (mon.2) 179 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:14:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: audit 2024-03-07T03:14:49.943066+0000 mon.c (mon.2) 180 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:14:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: audit 2024-03-07T03:14:50.986262+0000 mon.c (mon.2) 181 : audit 0 from='client.? 172.21.15.145:0/228987950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: cluster 2024-03-07T03:14:51.689979+0000 mgr.y (mgr.15132) 49 : cluster 0 pgmap v77: 137 pgs: 137 active+clean; 937 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 18 MiB/s wr, 246 op/s 2024-03-07T03:14:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: audit 2024-03-07T03:14:53.424116+0000 mon.c (mon.2) 182 : audit 0 from='client.? 172.21.15.145:0/2943588756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: cluster 2024-03-07T03:14:53.636200+0000 mon.a (mon.0) 1 : cluster 1 mon.a calling monitor election 2024-03-07T03:14:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: cluster 2024-03-07T03:14:53.660602+0000 mon.a (mon.0) 2 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:14:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: cluster 2024-03-07T03:14:53.671708+0000 mon.a (mon.0) 3 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:14:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: cluster 2024-03-07T03:14:53.691224+0000 mon.a (mon.0) 4 : cluster 0 fsmap foo:1 {0=foo.smithi189.zkinbe=up:active} 1 up:standby 2024-03-07T03:14:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: cluster 2024-03-07T03:14:53.691321+0000 mon.a (mon.0) 5 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2024-03-07T03:14:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: cluster 2024-03-07T03:14:53.693584+0000 mon.a (mon.0) 6 : cluster 0 mgrmap e31: y(active, since 56s), standbys: x 2024-03-07T03:14:54.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: cluster 2024-03-07T03:14:53.694809+0000 mon.a (mon.0) 7 : cluster 3 Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-03-07T03:14:54.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: cluster 2024-03-07T03:14:53.694850+0000 mon.a (mon.0) 8 : cluster 3 [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-07T03:14:54.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: cluster 2024-03-07T03:14:53.694876+0000 mon.a (mon.0) 9 : cluster 3 application not enabled on pool 'test-librbd-smithi145-68337-9' 2024-03-07T03:14:54.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: cluster 2024-03-07T03:14:53.694921+0000 mon.a (mon.0) 10 : cluster 3 use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-07T03:14:54.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: audit 2024-03-07T03:14:53.713351+0000 mon.a (mon.0) 11 : audit 1 from='mgr.15132 ' entity='' 2024-03-07T03:14:54.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[30906]: cluster 2024-03-07T03:14:53.714552+0000 mon.a (mon.0) 12 : cluster 0 mgrmap e32: y(active, since 56s), standbys: x 2024-03-07T03:14:54.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: debug 2024-03-07T03:14:53.743+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:14:54.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: audit 2024-03-07T03:14:49.939432+0000 mon.c (mon.2) 178 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:14:54.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: audit 2024-03-07T03:14:49.942759+0000 mon.c (mon.2) 179 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:14:54.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: audit 2024-03-07T03:14:49.943066+0000 mon.c (mon.2) 180 : audit 0 from='mgr.15132 172.21.15.145:0/3302128909' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:14:54.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: audit 2024-03-07T03:14:50.986262+0000 mon.c (mon.2) 181 : audit 0 from='client.? 172.21.15.145:0/228987950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:54.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: cluster 2024-03-07T03:14:51.689979+0000 mgr.y (mgr.15132) 49 : cluster 0 pgmap v77: 137 pgs: 137 active+clean; 937 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 18 MiB/s wr, 246 op/s 2024-03-07T03:14:54.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: audit 2024-03-07T03:14:53.424116+0000 mon.c (mon.2) 182 : audit 0 from='client.? 172.21.15.145:0/2943588756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:54.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: cluster 2024-03-07T03:14:53.636200+0000 mon.a (mon.0) 1 : cluster 1 mon.a calling monitor election 2024-03-07T03:14:54.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: cluster 2024-03-07T03:14:53.660602+0000 mon.a (mon.0) 2 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:14:54.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: cluster 2024-03-07T03:14:53.671708+0000 mon.a (mon.0) 3 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:14:54.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: cluster 2024-03-07T03:14:53.691224+0000 mon.a (mon.0) 4 : cluster 0 fsmap foo:1 {0=foo.smithi189.zkinbe=up:active} 1 up:standby 2024-03-07T03:14:54.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: cluster 2024-03-07T03:14:53.691321+0000 mon.a (mon.0) 5 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2024-03-07T03:14:54.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: cluster 2024-03-07T03:14:53.693584+0000 mon.a (mon.0) 6 : cluster 0 mgrmap e31: y(active, since 56s), standbys: x 2024-03-07T03:14:54.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: cluster 2024-03-07T03:14:53.694809+0000 mon.a (mon.0) 7 : cluster 3 Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-03-07T03:14:54.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: cluster 2024-03-07T03:14:53.694850+0000 mon.a (mon.0) 8 : cluster 3 [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-07T03:14:54.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: cluster 2024-03-07T03:14:53.694876+0000 mon.a (mon.0) 9 : cluster 3 application not enabled on pool 'test-librbd-smithi145-68337-9' 2024-03-07T03:14:54.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: cluster 2024-03-07T03:14:53.694921+0000 mon.a (mon.0) 10 : cluster 3 use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-07T03:14:54.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: audit 2024-03-07T03:14:53.713351+0000 mon.a (mon.0) 11 : audit 1 from='mgr.15132 ' entity='' 2024-03-07T03:14:54.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:53 smithi145 bash[79878]: cluster 2024-03-07T03:14:53.714552+0000 mon.a (mon.0) 12 : cluster 0 mgrmap e32: y(active, since 56s), standbys: x 2024-03-07T03:14:54.080 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[60207]: ignoring --setuser ceph since I am not root 2024-03-07T03:14:54.081 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[60207]: ignoring --setgroup ceph since I am not root 2024-03-07T03:14:54.081 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[60207]: debug 2024-03-07T03:14:53.780+0000 7fa731d79700 1 -- 172.21.15.189:0/2140257226 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564130e145a0 con 0x564131b94400 2024-03-07T03:14:54.081 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:53 smithi189 bash[60207]: debug 2024-03-07T03:14:53.924+0000 7fa73bb5d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:14:54.769 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap1 2024-03-07T03:14:54.834 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:14:55.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:54 smithi145 bash[79878]: cluster 2024-03-07T03:14:53.746362+0000 mon.a (mon.0) 13 : cluster 4 Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:14:55.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:54 smithi145 bash[79878]: cluster 2024-03-07T03:14:53.759945+0000 mon.a (mon.0) 14 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2024-03-07T03:14:55.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:54 smithi145 bash[30906]: cluster 2024-03-07T03:14:53.746362+0000 mon.a (mon.0) 13 : cluster 4 Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:14:55.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:54 smithi145 bash[30906]: cluster 2024-03-07T03:14:53.759945+0000 mon.a (mon.0) 14 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2024-03-07T03:14:55.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:54 smithi189 bash[23614]: cluster 2024-03-07T03:14:53.746362+0000 mon.a (mon.0) 13 : cluster 4 Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:14:55.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:54 smithi189 bash[23614]: cluster 2024-03-07T03:14:53.759945+0000 mon.a (mon.0) 14 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2024-03-07T03:14:55.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:55 smithi189 bash[60207]: debug 2024-03-07T03:14:55.504+0000 7fa73bb5d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:14:55.755 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:55 smithi189 bash[60207]: debug 2024-03-07T03:14:55.604+0000 7fa73bb5d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:14:55.808 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:55 smithi145 bash[72238]: debug 2024-03-07T03:14:55.567+0000 7f31cddeb200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:14:55.808 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:55 smithi145 bash[72238]: debug 2024-03-07T03:14:55.667+0000 7f31cddeb200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:14:55.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:55 smithi145 bash[79878]: cluster 2024-03-07T03:14:54.772242+0000 mon.a (mon.0) 15 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2024-03-07T03:14:55.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:55 smithi145 bash[30906]: cluster 2024-03-07T03:14:54.772242+0000 mon.a (mon.0) 15 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2024-03-07T03:14:55.860 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 1 2024-03-07T03:14:56.062 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:14:56.080 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:55 smithi189 bash[60207]: debug 2024-03-07T03:14:55.864+0000 7fa73bb5d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:14:56.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:55 smithi189 bash[23614]: cluster 2024-03-07T03:14:54.772242+0000 mon.a (mon.0) 15 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2024-03-07T03:14:56.124 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:55 smithi145 bash[72238]: debug 2024-03-07T03:14:55.959+0000 7f31cddeb200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:14:56.237 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:14:56.922 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch ps' 2024-03-07T03:14:56.958 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.CloneV1Parent (6861 ms) 2024-03-07T03:14:56.958 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.CloneV2Parent 2024-03-07T03:14:57.003 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:14:57.022 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 0~10 2024-03-07T03:14:57.037 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:14:57.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:56 smithi145 bash[72238]: debug 2024-03-07T03:14:56.831+0000 7f31cddeb200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:14:57.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:56 smithi145 bash[72238]: debug 2024-03-07T03:14:56.967+0000 7f31cddeb200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:14:57.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:56 smithi145 bash[30906]: cluster 2024-03-07T03:14:55.788513+0000 mon.a (mon.0) 16 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2024-03-07T03:14:57.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:56 smithi145 bash[30906]: audit 2024-03-07T03:14:55.858012+0000 mon.a (mon.0) 17 : audit 0 from='client.? 172.21.15.145:0/410216667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:57.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:56 smithi145 bash[79878]: cluster 2024-03-07T03:14:55.788513+0000 mon.a (mon.0) 16 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2024-03-07T03:14:57.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:56 smithi145 bash[79878]: audit 2024-03-07T03:14:55.858012+0000 mon.a (mon.0) 17 : audit 0 from='client.? 172.21.15.145:0/410216667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:57.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:56 smithi189 bash[23614]: cluster 2024-03-07T03:14:55.788513+0000 mon.a (mon.0) 16 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2024-03-07T03:14:57.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:56 smithi189 bash[23614]: audit 2024-03-07T03:14:55.858012+0000 mon.a (mon.0) 17 : audit 0 from='client.? 172.21.15.145:0/410216667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:57.080 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:56 smithi189 bash[60207]: debug 2024-03-07T03:14:56.708+0000 7fa73bb5d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:14:57.081 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:56 smithi189 bash[60207]: debug 2024-03-07T03:14:56.832+0000 7fa73bb5d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:14:57.081 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:56 smithi189 bash[60207]: debug 2024-03-07T03:14:56.948+0000 7fa73bb5d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:14:57.308 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:57 smithi145 bash[72238]: debug 2024-03-07T03:14:57.091+0000 7f31cddeb200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:14:57.580 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:57 smithi189 bash[60207]: debug 2024-03-07T03:14:57.320+0000 7fa73bb5d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:14:57.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:57 smithi189 bash[60207]: debug 2024-03-07T03:14:57.436+0000 7fa73bb5d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:14:57.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:57 smithi189 bash[60207]: debug 2024-03-07T03:14:57.544+0000 7fa73bb5d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:14:57.781 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:57 smithi145 bash[72238]: debug 2024-03-07T03:14:57.475+0000 7f31cddeb200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:14:57.782 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:57 smithi145 bash[72238]: debug 2024-03-07T03:14:57.591+0000 7f31cddeb200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:14:57.782 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:57 smithi145 bash[72238]: debug 2024-03-07T03:14:57.699+0000 7f31cddeb200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:14:58.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:57 smithi145 bash[79878]: cluster 2024-03-07T03:14:56.789584+0000 mon.a (mon.0) 18 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2024-03-07T03:14:58.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:57 smithi145 bash[79878]: cluster 2024-03-07T03:14:57.230191+0000 mon.a (mon.0) 19 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2024-03-07T03:14:58.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:57 smithi145 bash[30906]: cluster 2024-03-07T03:14:56.789584+0000 mon.a (mon.0) 18 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2024-03-07T03:14:58.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:57 smithi145 bash[30906]: cluster 2024-03-07T03:14:57.230191+0000 mon.a (mon.0) 19 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2024-03-07T03:14:58.080 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:57 smithi189 bash[60207]: debug 2024-03-07T03:14:57.908+0000 7fa73bb5d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:14:58.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:57 smithi189 bash[23614]: cluster 2024-03-07T03:14:56.789584+0000 mon.a (mon.0) 18 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2024-03-07T03:14:58.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:57 smithi189 bash[23614]: cluster 2024-03-07T03:14:57.230191+0000 mon.a (mon.0) 19 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2024-03-07T03:14:58.243 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap1 2024-03-07T03:14:58.323 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:58 smithi145 bash[72238]: debug 2024-03-07T03:14:58.075+0000 7f31cddeb200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:14:58.359 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:14:58.435 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:58 smithi189 bash[60207]: debug 2024-03-07T03:14:58.156+0000 7fa73bb5d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:14:58.625 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:58 smithi145 bash[72238]: debug 2024-03-07T03:14:58.331+0000 7f31cddeb200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:14:58.830 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:58 smithi189 bash[60207]: debug 2024-03-07T03:14:58.444+0000 7fa73bb5d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:14:58.831 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:58 smithi189 bash[60207]: debug 2024-03-07T03:14:58.560+0000 7fa73bb5d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:14:59.048 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:58 smithi145 bash[72238]: debug 2024-03-07T03:14:58.635+0000 7f31cddeb200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:14:59.048 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:58 smithi145 bash[72238]: debug 2024-03-07T03:14:58.763+0000 7f31cddeb200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:14:59.227 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:58 smithi189 bash[60207]: debug 2024-03-07T03:14:58.840+0000 7fa73bb5d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:14:59.228 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:59 smithi189 bash[60207]: debug 2024-03-07T03:14:59.092+0000 7fa73bb5d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:14:59.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:14:59 smithi145 bash[79878]: cluster 2024-03-07T03:14:58.231074+0000 mon.a (mon.0) 20 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2024-03-07T03:14:59.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:14:59 smithi145 bash[30906]: cluster 2024-03-07T03:14:58.231074+0000 mon.a (mon.0) 20 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2024-03-07T03:14:59.308 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:59 smithi145 bash[72238]: debug 2024-03-07T03:14:59.055+0000 7f31cddeb200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:14:59.514 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:14:59 smithi189 bash[23614]: cluster 2024-03-07T03:14:58.231074+0000 mon.a (mon.0) 20 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2024-03-07T03:14:59.613 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:14:59.707 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:14:59.795 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:59 smithi145 bash[72238]: debug 2024-03-07T03:14:59.319+0000 7f31cddeb200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:14:59.830 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:59 smithi189 bash[60207]: debug 2024-03-07T03:14:59.524+0000 7fa73bb5d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:14:59.830 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:59 smithi189 bash[60207]: debug 2024-03-07T03:14:59.648+0000 7fa73bb5d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:15:00.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:59 smithi145 bash[72238]: debug 2024-03-07T03:14:59.803+0000 7f31cddeb200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:15:00.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:14:59 smithi145 bash[72238]: debug 2024-03-07T03:14:59.927+0000 7f31cddeb200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:15:00.242 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:14:59 smithi189 bash[60207]: debug 2024-03-07T03:14:59.912+0000 7fa73bb5d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:15:00.426 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.CloneV2Parent (3467 ms) 2024-03-07T03:15:00.426 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.CloneV1ParentAbort 2024-03-07T03:15:00.499 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:15:00.523 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 0~10 2024-03-07T03:15:00.542 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:15:00.558 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:00 smithi145 bash[72238]: debug 2024-03-07T03:15:00.203+0000 7f31cddeb200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:15:00.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:00 smithi145 bash[79878]: cluster 2024-03-07T03:14:59.235638+0000 mon.a (mon.0) 21 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2024-03-07T03:15:00.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:00 smithi145 bash[30906]: cluster 2024-03-07T03:14:59.235638+0000 mon.a (mon.0) 21 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2024-03-07T03:15:00.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:00 smithi189 bash[23614]: cluster 2024-03-07T03:14:59.235638+0000 mon.a (mon.0) 21 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2024-03-07T03:15:01.254 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:15:00 smithi189 bash[60207]: debug 2024-03-07T03:15:00.964+0000 7fa73bb5d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:15:01.255 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:15:01 smithi189 bash[60207]: debug 2024-03-07T03:15:01.080+0000 7fa73bb5d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:15:01.308 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:01 smithi145 bash[72238]: debug 2024-03-07T03:15:01.295+0000 7f31cddeb200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:15:01.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:01 smithi145 bash[79878]: cluster 2024-03-07T03:15:00.248977+0000 mon.a (mon.0) 22 : cluster 0 osdmap e279: 8 total, 8 up, 8 in 2024-03-07T03:15:01.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:01 smithi145 bash[30906]: cluster 2024-03-07T03:15:00.248977+0000 mon.a (mon.0) 22 : cluster 0 osdmap e279: 8 total, 8 up, 8 in 2024-03-07T03:15:01.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:01 smithi189 bash[23614]: cluster 2024-03-07T03:15:00.248977+0000 mon.a (mon.0) 22 : cluster 0 osdmap e279: 8 total, 8 up, 8 in 2024-03-07T03:15:01.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:15:01 smithi189 bash[60207]: debug 2024-03-07T03:15:01.384+0000 7fa73bb5d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:15:01.581 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:15:01 smithi189 bash[60207]: debug 2024-03-07T03:15:01.500+0000 7fa73bb5d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:15:01.627 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:15:01.734 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:01 smithi145 bash[72238]: debug 2024-03-07T03:15:01.427+0000 7f31cddeb200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:15:02.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:01 smithi145 bash[72238]: debug 2024-03-07T03:15:01.743+0000 7f31cddeb200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:15:02.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:01 smithi145 bash[72238]: debug 2024-03-07T03:15:01.859+0000 7f31cddeb200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:15:02.073 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:15:01 smithi189 bash[60207]: debug 2024-03-07T03:15:01.960+0000 7fa73bb5d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:15:02.308 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap1 2024-03-07T03:15:02.330 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[59814]: ts=2024-03-07T03:15:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:15:02.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[60207]: debug 2024-03-07T03:15:02.080+0000 7fa73bb5d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:15:02.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[60207]: [07/Mar/2024:03:15:02] ENGINE Bus STARTING 2024-03-07T03:15:02.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[60207]: CherryPy Checker: 2024-03-07T03:15:02.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[60207]: The Application mounted at '' has an empty config. 2024-03-07T03:15:02.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[60207]: [07/Mar/2024:03:15:02] ENGINE Serving on http://:::9283 2024-03-07T03:15:02.331 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[60207]: [07/Mar/2024:03:15:02] ENGINE Bus STARTED 2024-03-07T03:15:02.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[23614]: cluster 2024-03-07T03:15:01.265005+0000 mon.a (mon.0) 23 : cluster 0 osdmap e280: 8 total, 8 up, 8 in 2024-03-07T03:15:02.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[23614]: audit 2024-03-07T03:15:02.110541+0000 mon.b (mon.1) 89 : audit 0 from='mgr.? 172.21.15.189:0/1069228285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:15:02.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[23614]: cluster 2024-03-07T03:15:02.110971+0000 mon.a (mon.0) 24 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:15:02.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[23614]: cluster 2024-03-07T03:15:02.111141+0000 mon.a (mon.0) 25 : cluster 0 Standby manager daemon x started 2024-03-07T03:15:02.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[23614]: audit 2024-03-07T03:15:02.111783+0000 mon.b (mon.1) 90 : audit 0 from='mgr.? 172.21.15.189:0/1069228285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:15:02.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[23614]: audit 2024-03-07T03:15:02.114934+0000 mon.b (mon.1) 91 : audit 0 from='mgr.? 172.21.15.189:0/1069228285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:15:02.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:02 smithi189 bash[23614]: audit 2024-03-07T03:15:02.115463+0000 mon.b (mon.1) 92 : audit 0 from='mgr.? 172.21.15.189:0/1069228285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:15:02.453 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:15:02.558 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: debug 2024-03-07T03:15:02.339+0000 7f31cddeb200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:15:02.558 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: debug 2024-03-07T03:15:02.471+0000 7f31cddeb200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:15:02.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[30906]: cluster 2024-03-07T03:15:01.265005+0000 mon.a (mon.0) 23 : cluster 0 osdmap e280: 8 total, 8 up, 8 in 2024-03-07T03:15:02.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[30906]: audit 2024-03-07T03:15:02.110541+0000 mon.b (mon.1) 89 : audit 0 from='mgr.? 172.21.15.189:0/1069228285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:15:02.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[30906]: cluster 2024-03-07T03:15:02.110971+0000 mon.a (mon.0) 24 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:15:02.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[30906]: cluster 2024-03-07T03:15:02.111141+0000 mon.a (mon.0) 25 : cluster 0 Standby manager daemon x started 2024-03-07T03:15:02.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[30906]: audit 2024-03-07T03:15:02.111783+0000 mon.b (mon.1) 90 : audit 0 from='mgr.? 172.21.15.189:0/1069228285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:15:02.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[30906]: audit 2024-03-07T03:15:02.114934+0000 mon.b (mon.1) 91 : audit 0 from='mgr.? 172.21.15.189:0/1069228285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:15:02.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[30906]: audit 2024-03-07T03:15:02.115463+0000 mon.b (mon.1) 92 : audit 0 from='mgr.? 172.21.15.189:0/1069228285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:15:02.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[79878]: cluster 2024-03-07T03:15:01.265005+0000 mon.a (mon.0) 23 : cluster 0 osdmap e280: 8 total, 8 up, 8 in 2024-03-07T03:15:02.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[79878]: audit 2024-03-07T03:15:02.110541+0000 mon.b (mon.1) 89 : audit 0 from='mgr.? 172.21.15.189:0/1069228285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:15:02.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[79878]: cluster 2024-03-07T03:15:02.110971+0000 mon.a (mon.0) 24 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:15:02.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[79878]: cluster 2024-03-07T03:15:02.111141+0000 mon.a (mon.0) 25 : cluster 0 Standby manager daemon x started 2024-03-07T03:15:02.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[79878]: audit 2024-03-07T03:15:02.111783+0000 mon.b (mon.1) 90 : audit 0 from='mgr.? 172.21.15.189:0/1069228285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:15:02.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[79878]: audit 2024-03-07T03:15:02.114934+0000 mon.b (mon.1) 91 : audit 0 from='mgr.? 172.21.15.189:0/1069228285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:15:02.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[79878]: audit 2024-03-07T03:15:02.115463+0000 mon.b (mon.1) 92 : audit 0 from='mgr.? 172.21.15.189:0/1069228285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:15:02.960 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: [07/Mar/2024:03:15:02] ENGINE Bus STARTING 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: debug 2024-03-07T03:15:02.683+0000 7f312c169700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: debug 2024-03-07T03:15:02.683+0000 7f312c169700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: debug 2024-03-07T03:15:02.683+0000 7f312c169700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: debug 2024-03-07T03:15:02.683+0000 7f312c169700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: debug 2024-03-07T03:15:02.683+0000 7f312c169700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: debug 2024-03-07T03:15:02.683+0000 7f3131974700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: debug 2024-03-07T03:15:02.683+0000 7f3131974700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: debug 2024-03-07T03:15:02.683+0000 7f3131974700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: debug 2024-03-07T03:15:02.687+0000 7f3131974700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: debug 2024-03-07T03:15:02.687+0000 7f3131974700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:15:02] "GET /metrics HTTP/1.1" 200 35883 "" "Prometheus/2.43.0" 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: CherryPy Checker: 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: The Application mounted at '' has an empty config. 2024-03-07T03:15:02.961 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: [07/Mar/2024:03:15:02] ENGINE Serving on http://:::9283 2024-03-07T03:15:02.962 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:02 smithi145 bash[72238]: [07/Mar/2024:03:15:02] ENGINE Bus STARTED 2024-03-07T03:15:03.558 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[72238]: debug 2024-03-07T03:15:03.523+0000 7f318c55a700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-03-07T03:15:03.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: cluster 2024-03-07T03:15:02.286089+0000 mon.a (mon.0) 26 : cluster 0 mgrmap e33: y(active, since 64s), standbys: x 2024-03-07T03:15:03.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: cluster 2024-03-07T03:15:02.286176+0000 mon.a (mon.0) 27 : cluster 0 osdmap e281: 8 total, 8 up, 8 in 2024-03-07T03:15:03.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: cluster 2024-03-07T03:15:02.495951+0000 mon.a (mon.0) 28 : cluster 1 Active manager daemon y restarted 2024-03-07T03:15:03.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: cluster 2024-03-07T03:15:02.497309+0000 mon.a (mon.0) 29 : cluster 1 Activating manager daemon y 2024-03-07T03:15:03.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: cluster 2024-03-07T03:15:02.504714+0000 mon.a (mon.0) 30 : cluster 0 osdmap e282: 8 total, 8 up, 8 in 2024-03-07T03:15:03.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: cluster 2024-03-07T03:15:02.505102+0000 mon.a (mon.0) 31 : cluster 0 mgrmap e34: y(active, starting, since 0.00816438s), standbys: x 2024-03-07T03:15:03.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.510219+0000 mon.a (mon.0) 32 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:15:03.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.510472+0000 mon.a (mon.0) 33 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:15:03.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.510671+0000 mon.a (mon.0) 34 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:15:03.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.512145+0000 mon.a (mon.0) 35 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:15:03.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.512404+0000 mon.a (mon.0) 36 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.zkinbe"}]: dispatch 2024-03-07T03:15:03.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.512653+0000 mon.a (mon.0) 37 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:15:03.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.512827+0000 mon.a (mon.0) 38 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:15:03.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.513051+0000 mon.a (mon.0) 39 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.513382+0000 mon.a (mon.0) 40 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.513783+0000 mon.a (mon.0) 41 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.514191+0000 mon.a (mon.0) 42 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.514544+0000 mon.a (mon.0) 43 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.514821+0000 mon.a (mon.0) 44 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.515042+0000 mon.a (mon.0) 45 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.515286+0000 mon.a (mon.0) 46 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.515546+0000 mon.a (mon.0) 47 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.515909+0000 mon.a (mon.0) 48 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.516610+0000 mon.a (mon.0) 49 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: cluster 2024-03-07T03:15:02.544940+0000 mon.a (mon.0) 50 : cluster 1 Manager daemon y is now available 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.592440+0000 mon.a (mon.0) 51 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.597617+0000 mon.a (mon.0) 52 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.627521+0000 mon.a (mon.0) 53 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:03.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.628777+0000 mon.a (mon.0) 54 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:03.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.634691+0000 mon.a (mon.0) 55 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:15:03.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[30906]: audit 2024-03-07T03:15:02.673745+0000 mon.a (mon.0) 56 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:15:03.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: cluster 2024-03-07T03:15:02.286089+0000 mon.a (mon.0) 26 : cluster 0 mgrmap e33: y(active, since 64s), standbys: x 2024-03-07T03:15:03.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: cluster 2024-03-07T03:15:02.286176+0000 mon.a (mon.0) 27 : cluster 0 osdmap e281: 8 total, 8 up, 8 in 2024-03-07T03:15:03.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: cluster 2024-03-07T03:15:02.495951+0000 mon.a (mon.0) 28 : cluster 1 Active manager daemon y restarted 2024-03-07T03:15:03.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: cluster 2024-03-07T03:15:02.497309+0000 mon.a (mon.0) 29 : cluster 1 Activating manager daemon y 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: cluster 2024-03-07T03:15:02.504714+0000 mon.a (mon.0) 30 : cluster 0 osdmap e282: 8 total, 8 up, 8 in 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: cluster 2024-03-07T03:15:02.505102+0000 mon.a (mon.0) 31 : cluster 0 mgrmap e34: y(active, starting, since 0.00816438s), standbys: x 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.510219+0000 mon.a (mon.0) 32 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.510472+0000 mon.a (mon.0) 33 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.510671+0000 mon.a (mon.0) 34 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.512145+0000 mon.a (mon.0) 35 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.512404+0000 mon.a (mon.0) 36 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.zkinbe"}]: dispatch 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.512653+0000 mon.a (mon.0) 37 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.512827+0000 mon.a (mon.0) 38 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.513051+0000 mon.a (mon.0) 39 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.513382+0000 mon.a (mon.0) 40 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.513783+0000 mon.a (mon.0) 41 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.514191+0000 mon.a (mon.0) 42 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.514544+0000 mon.a (mon.0) 43 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:15:03.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.514821+0000 mon.a (mon.0) 44 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:15:03.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.515042+0000 mon.a (mon.0) 45 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:15:03.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.515286+0000 mon.a (mon.0) 46 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:15:03.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.515546+0000 mon.a (mon.0) 47 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:15:03.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.515909+0000 mon.a (mon.0) 48 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:15:03.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.516610+0000 mon.a (mon.0) 49 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:15:03.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: cluster 2024-03-07T03:15:02.544940+0000 mon.a (mon.0) 50 : cluster 1 Manager daemon y is now available 2024-03-07T03:15:03.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.592440+0000 mon.a (mon.0) 51 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:03.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.597617+0000 mon.a (mon.0) 52 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:03.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.627521+0000 mon.a (mon.0) 53 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:03.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.628777+0000 mon.a (mon.0) 54 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:03.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.634691+0000 mon.a (mon.0) 55 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:15:03.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:03 smithi145 bash[79878]: audit 2024-03-07T03:15:02.673745+0000 mon.a (mon.0) 56 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:15:03.653 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[59814]: ts=2024-03-07T03:15:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:15:03.653 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: cluster 2024-03-07T03:15:02.286089+0000 mon.a (mon.0) 26 : cluster 0 mgrmap e33: y(active, since 64s), standbys: x 2024-03-07T03:15:03.653 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: cluster 2024-03-07T03:15:02.286176+0000 mon.a (mon.0) 27 : cluster 0 osdmap e281: 8 total, 8 up, 8 in 2024-03-07T03:15:03.653 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: cluster 2024-03-07T03:15:02.495951+0000 mon.a (mon.0) 28 : cluster 1 Active manager daemon y restarted 2024-03-07T03:15:03.653 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: cluster 2024-03-07T03:15:02.497309+0000 mon.a (mon.0) 29 : cluster 1 Activating manager daemon y 2024-03-07T03:15:03.653 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: cluster 2024-03-07T03:15:02.504714+0000 mon.a (mon.0) 30 : cluster 0 osdmap e282: 8 total, 8 up, 8 in 2024-03-07T03:15:03.654 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: cluster 2024-03-07T03:15:02.505102+0000 mon.a (mon.0) 31 : cluster 0 mgrmap e34: y(active, starting, since 0.00816438s), standbys: x 2024-03-07T03:15:03.654 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.510219+0000 mon.a (mon.0) 32 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:15:03.654 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.510472+0000 mon.a (mon.0) 33 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:15:03.654 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.510671+0000 mon.a (mon.0) 34 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:15:03.654 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.512145+0000 mon.a (mon.0) 35 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi145.zmgvgw"}]: dispatch 2024-03-07T03:15:03.654 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.512404+0000 mon.a (mon.0) 36 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi189.zkinbe"}]: dispatch 2024-03-07T03:15:03.654 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.512653+0000 mon.a (mon.0) 37 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:15:03.654 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.512827+0000 mon.a (mon.0) 38 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:15:03.654 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.513051+0000 mon.a (mon.0) 39 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:15:03.654 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.513382+0000 mon.a (mon.0) 40 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:15:03.654 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.513783+0000 mon.a (mon.0) 41 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:15:03.654 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.514191+0000 mon.a (mon.0) 42 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:15:03.654 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.514544+0000 mon.a (mon.0) 43 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:15:03.655 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.514821+0000 mon.a (mon.0) 44 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:15:03.655 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.515042+0000 mon.a (mon.0) 45 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:15:03.655 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.515286+0000 mon.a (mon.0) 46 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:15:03.655 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.515546+0000 mon.a (mon.0) 47 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:15:03.655 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.515909+0000 mon.a (mon.0) 48 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:15:03.655 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.516610+0000 mon.a (mon.0) 49 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:15:03.655 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: cluster 2024-03-07T03:15:02.544940+0000 mon.a (mon.0) 50 : cluster 1 Manager daemon y is now available 2024-03-07T03:15:03.655 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.592440+0000 mon.a (mon.0) 51 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:03.655 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.597617+0000 mon.a (mon.0) 52 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:03.655 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.627521+0000 mon.a (mon.0) 53 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:03.655 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.628777+0000 mon.a (mon.0) 54 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:03.656 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.634691+0000 mon.a (mon.0) 55 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:15:03.656 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:03 smithi189 bash[23614]: audit 2024-03-07T03:15:02.673745+0000 mon.a (mon.0) 56 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:15:03.846 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:15:04.746 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.CloneV1ParentAbort (4320 ms) 2024-03-07T03:15:04.746 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.CloneV2ParentAbort 2024-03-07T03:15:04.806 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:15:04.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[79878]: cephadm 2024-03-07T03:15:03.080302+0000 mgr.y (mgr.34107) 1 : cephadm 1 [07/Mar/2024:03:15:03] ENGINE Bus STARTING 2024-03-07T03:15:04.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[79878]: cephadm 2024-03-07T03:15:03.195513+0000 mgr.y (mgr.34107) 2 : cephadm 1 [07/Mar/2024:03:15:03] ENGINE Serving on https://172.21.15.145:7150 2024-03-07T03:15:04.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[79878]: cephadm 2024-03-07T03:15:03.297480+0000 mgr.y (mgr.34107) 3 : cephadm 1 [07/Mar/2024:03:15:03] ENGINE Serving on http://172.21.15.145:8765 2024-03-07T03:15:04.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[79878]: cephadm 2024-03-07T03:15:03.298038+0000 mgr.y (mgr.34107) 4 : cephadm 1 [07/Mar/2024:03:15:03] ENGINE Bus STARTED 2024-03-07T03:15:04.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[79878]: cluster 2024-03-07T03:15:03.524249+0000 mon.a (mon.0) 57 : cluster 0 mgrmap e35: y(active, since 1.02729s), standbys: x 2024-03-07T03:15:04.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[79878]: cluster 2024-03-07T03:15:03.531795+0000 mon.a (mon.0) 58 : cluster 0 osdmap e283: 8 total, 8 up, 8 in 2024-03-07T03:15:04.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[79878]: cluster 2024-03-07T03:15:03.533654+0000 mgr.y (mgr.34107) 5 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 278 GiB/s rd, 109 GiB/s wr, 5.58M op/s; 261 MiB/s, 927 objects/s recovering 2024-03-07T03:15:04.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[30906]: cephadm 2024-03-07T03:15:03.080302+0000 mgr.y (mgr.34107) 1 : cephadm 1 [07/Mar/2024:03:15:03] ENGINE Bus STARTING 2024-03-07T03:15:04.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[30906]: cephadm 2024-03-07T03:15:03.195513+0000 mgr.y (mgr.34107) 2 : cephadm 1 [07/Mar/2024:03:15:03] ENGINE Serving on https://172.21.15.145:7150 2024-03-07T03:15:04.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[30906]: cephadm 2024-03-07T03:15:03.297480+0000 mgr.y (mgr.34107) 3 : cephadm 1 [07/Mar/2024:03:15:03] ENGINE Serving on http://172.21.15.145:8765 2024-03-07T03:15:04.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[30906]: cephadm 2024-03-07T03:15:03.298038+0000 mgr.y (mgr.34107) 4 : cephadm 1 [07/Mar/2024:03:15:03] ENGINE Bus STARTED 2024-03-07T03:15:04.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[30906]: cluster 2024-03-07T03:15:03.524249+0000 mon.a (mon.0) 57 : cluster 0 mgrmap e35: y(active, since 1.02729s), standbys: x 2024-03-07T03:15:04.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[30906]: cluster 2024-03-07T03:15:03.531795+0000 mon.a (mon.0) 58 : cluster 0 osdmap e283: 8 total, 8 up, 8 in 2024-03-07T03:15:04.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:04 smithi145 bash[30906]: cluster 2024-03-07T03:15:03.533654+0000 mgr.y (mgr.34107) 5 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 278 GiB/s rd, 109 GiB/s wr, 5.58M op/s; 261 MiB/s, 927 objects/s recovering 2024-03-07T03:15:04.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:04 smithi189 bash[23614]: cephadm 2024-03-07T03:15:03.080302+0000 mgr.y (mgr.34107) 1 : cephadm 1 [07/Mar/2024:03:15:03] ENGINE Bus STARTING 2024-03-07T03:15:04.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:04 smithi189 bash[23614]: cephadm 2024-03-07T03:15:03.195513+0000 mgr.y (mgr.34107) 2 : cephadm 1 [07/Mar/2024:03:15:03] ENGINE Serving on https://172.21.15.145:7150 2024-03-07T03:15:04.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:04 smithi189 bash[23614]: cephadm 2024-03-07T03:15:03.297480+0000 mgr.y (mgr.34107) 3 : cephadm 1 [07/Mar/2024:03:15:03] ENGINE Serving on http://172.21.15.145:8765 2024-03-07T03:15:04.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:04 smithi189 bash[23614]: cephadm 2024-03-07T03:15:03.298038+0000 mgr.y (mgr.34107) 4 : cephadm 1 [07/Mar/2024:03:15:03] ENGINE Bus STARTED 2024-03-07T03:15:04.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:04 smithi189 bash[23614]: cluster 2024-03-07T03:15:03.524249+0000 mon.a (mon.0) 57 : cluster 0 mgrmap e35: y(active, since 1.02729s), standbys: x 2024-03-07T03:15:04.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:04 smithi189 bash[23614]: cluster 2024-03-07T03:15:03.531795+0000 mon.a (mon.0) 58 : cluster 0 osdmap e283: 8 total, 8 up, 8 in 2024-03-07T03:15:04.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:04 smithi189 bash[23614]: cluster 2024-03-07T03:15:03.533654+0000 mgr.y (mgr.34107) 5 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 278 GiB/s rd, 109 GiB/s wr, 5.58M op/s; 261 MiB/s, 927 objects/s recovering 2024-03-07T03:15:04.834 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 0~10 2024-03-07T03:15:04.859 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:15:05.401 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-07T03:15:05.401 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (7m) 61s ago 8m 14.6M - 0.25.0 c8568f914cd2 cfa1e64a6007 2024-03-07T03:15:05.401 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi189 *:3000 running (8m) 31s ago 8m 79.6M - 9.4.7 954c08fa6188 89badf11bd11 2024-03-07T03:15:05.401 INFO:teuthology.orchestra.run.smithi145.stdout:mds.foo.smithi145.zmgvgw smithi145 running (6m) 61s ago 6m 13.5M - 18.2.1-718-gae75fc9d 0164bb8a7c67 4ded79fef80e 2024-03-07T03:15:05.401 INFO:teuthology.orchestra.run.smithi145.stdout:mds.foo.smithi189.zkinbe smithi189 running (6m) 31s ago 6m 16.0M - 18.2.1-718-gae75fc9d 0164bb8a7c67 6589deb5c181 2024-03-07T03:15:05.401 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi189 *:8443,9283,8765 running (37s) 31s ago 16m 134M - 19.0.0-1905-gea9a5688 fdb83767c1ab acef232ce9b9 2024-03-07T03:15:05.401 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (85s) 61s ago 19m 468M - 19.0.0-1905-gea9a5688 fdb83767c1ab 2cae6fe389e0 2024-03-07T03:15:05.401 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (19m) 61s ago 19m 61.5M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 ba6b29c5069f 2024-03-07T03:15:05.401 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi189 running (17m) 31s ago 17m 47.3M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 5746e42ef7c9 2024-03-07T03:15:05.401 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (17m) 61s ago 17m 49.4M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 eb9564943cff 2024-03-07T03:15:05.401 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (8m) 61s ago 8m 8683k - 1.5.0 0da6a335fe13 895e660f39b6 2024-03-07T03:15:05.401 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi189 *:9100 running (8m) 31s ago 8m 8731k - 1.5.0 0da6a335fe13 ab7ffb60c1e8 2024-03-07T03:15:05.401 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (15m) 61s ago 16m 124M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 4485c22f1548 2024-03-07T03:15:05.402 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (15m) 61s ago 15m 103M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 f21d0bf7ebc8 2024-03-07T03:15:05.402 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (14m) 61s ago 14m 96.2M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 9e8c1bf873ba 2024-03-07T03:15:05.402 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (13m) 61s ago 13m 98.5M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 c209b1a43303 2024-03-07T03:15:05.402 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi189 running (12m) 31s ago 12m 109M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 8845b4f5216a 2024-03-07T03:15:05.402 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi189 running (11m) 31s ago 11m 96.6M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 44e5411eb07c 2024-03-07T03:15:05.402 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi189 running (10m) 31s ago 10m 122M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 c2aeea0466c7 2024-03-07T03:15:05.402 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi189 running (10m) 31s ago 10m 95.2M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 bf101bfb6800 2024-03-07T03:15:05.402 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi189 *:9095 running (44s) 31s ago 9m 37.1M - 2.43.0 a07b618ecd1d 144e5cadeea3 2024-03-07T03:15:05.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:05 smithi145 bash[79878]: cluster 2024-03-07T03:15:04.514862+0000 mgr.y (mgr.34107) 6 : cluster 0 pgmap v5: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.8 GiB/s rd, 730 MiB/s wr, 36.55k op/s; 1.7 MiB/s, 6 objects/s recovering 2024-03-07T03:15:05.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:05 smithi145 bash[79878]: cluster 2024-03-07T03:15:04.540426+0000 mon.a (mon.0) 59 : cluster 0 osdmap e284: 8 total, 8 up, 8 in 2024-03-07T03:15:05.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:05 smithi145 bash[79878]: cluster 2024-03-07T03:15:05.517447+0000 mon.a (mon.0) 60 : cluster 0 mgrmap e36: y(active, since 3s), standbys: x 2024-03-07T03:15:05.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:05 smithi145 bash[30906]: cluster 2024-03-07T03:15:04.514862+0000 mgr.y (mgr.34107) 6 : cluster 0 pgmap v5: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.8 GiB/s rd, 730 MiB/s wr, 36.55k op/s; 1.7 MiB/s, 6 objects/s recovering 2024-03-07T03:15:05.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:05 smithi145 bash[30906]: cluster 2024-03-07T03:15:04.540426+0000 mon.a (mon.0) 59 : cluster 0 osdmap e284: 8 total, 8 up, 8 in 2024-03-07T03:15:05.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:05 smithi145 bash[30906]: cluster 2024-03-07T03:15:05.517447+0000 mon.a (mon.0) 60 : cluster 0 mgrmap e36: y(active, since 3s), standbys: x 2024-03-07T03:15:05.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:05 smithi189 bash[23614]: cluster 2024-03-07T03:15:04.514862+0000 mgr.y (mgr.34107) 6 : cluster 0 pgmap v5: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 1.8 GiB/s rd, 730 MiB/s wr, 36.55k op/s; 1.7 MiB/s, 6 objects/s recovering 2024-03-07T03:15:05.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:05 smithi189 bash[23614]: cluster 2024-03-07T03:15:04.540426+0000 mon.a (mon.0) 59 : cluster 0 osdmap e284: 8 total, 8 up, 8 in 2024-03-07T03:15:05.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:05 smithi189 bash[23614]: cluster 2024-03-07T03:15:05.517447+0000 mon.a (mon.0) 60 : cluster 0 mgrmap e36: y(active, since 3s), standbys: x 2024-03-07T03:15:06.268 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade pause' 2024-03-07T03:15:06.571 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap1 2024-03-07T03:15:06.682 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:15:06.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:06 smithi145 bash[79878]: audit 2024-03-07T03:15:05.381081+0000 mgr.y (mgr.34107) 7 : audit 0 from='client.25313 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:15:06.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:06 smithi145 bash[79878]: cluster 2024-03-07T03:15:05.543320+0000 mon.a (mon.0) 61 : cluster 0 osdmap e285: 8 total, 8 up, 8 in 2024-03-07T03:15:06.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:06 smithi145 bash[30906]: audit 2024-03-07T03:15:05.381081+0000 mgr.y (mgr.34107) 7 : audit 0 from='client.25313 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:15:06.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:06 smithi145 bash[30906]: cluster 2024-03-07T03:15:05.543320+0000 mon.a (mon.0) 61 : cluster 0 osdmap e285: 8 total, 8 up, 8 in 2024-03-07T03:15:06.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:06 smithi189 bash[23614]: audit 2024-03-07T03:15:05.381081+0000 mgr.y (mgr.34107) 7 : audit 0 from='client.25313 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:15:06.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:06 smithi189 bash[23614]: cluster 2024-03-07T03:15:05.543320+0000 mon.a (mon.0) 61 : cluster 0 osdmap e285: 8 total, 8 up, 8 in 2024-03-07T03:15:07.620 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:15:07.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:07 smithi145 bash[79878]: cluster 2024-03-07T03:15:06.515399+0000 mgr.y (mgr.34107) 8 : cluster 0 pgmap v8: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 615 MiB/s rd, 241 MiB/s wr, 12.08k op/s; 579 KiB/s, 2 objects/s recovering 2024-03-07T03:15:07.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:07 smithi145 bash[79878]: cluster 2024-03-07T03:15:06.560907+0000 mon.a (mon.0) 62 : cluster 0 osdmap e286: 8 total, 8 up, 8 in 2024-03-07T03:15:07.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:07 smithi145 bash[79878]: cluster 2024-03-07T03:15:06.561509+0000 mon.a (mon.0) 63 : cluster 0 mgrmap e37: y(active, since 4s), standbys: x 2024-03-07T03:15:07.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:07 smithi145 bash[79878]: cluster 2024-03-07T03:15:07.233093+0000 mon.a (mon.0) 64 : cluster 0 osdmap e287: 8 total, 8 up, 8 in 2024-03-07T03:15:07.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:07 smithi145 bash[30906]: cluster 2024-03-07T03:15:06.515399+0000 mgr.y (mgr.34107) 8 : cluster 0 pgmap v8: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 615 MiB/s rd, 241 MiB/s wr, 12.08k op/s; 579 KiB/s, 2 objects/s recovering 2024-03-07T03:15:07.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:07 smithi145 bash[30906]: cluster 2024-03-07T03:15:06.560907+0000 mon.a (mon.0) 62 : cluster 0 osdmap e286: 8 total, 8 up, 8 in 2024-03-07T03:15:07.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:07 smithi145 bash[30906]: cluster 2024-03-07T03:15:06.561509+0000 mon.a (mon.0) 63 : cluster 0 mgrmap e37: y(active, since 4s), standbys: x 2024-03-07T03:15:07.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:07 smithi145 bash[30906]: cluster 2024-03-07T03:15:07.233093+0000 mon.a (mon.0) 64 : cluster 0 osdmap e287: 8 total, 8 up, 8 in 2024-03-07T03:15:07.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:07 smithi189 bash[23614]: cluster 2024-03-07T03:15:06.515399+0000 mgr.y (mgr.34107) 8 : cluster 0 pgmap v8: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 615 MiB/s rd, 241 MiB/s wr, 12.08k op/s; 579 KiB/s, 2 objects/s recovering 2024-03-07T03:15:07.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:07 smithi189 bash[23614]: cluster 2024-03-07T03:15:06.560907+0000 mon.a (mon.0) 62 : cluster 0 osdmap e286: 8 total, 8 up, 8 in 2024-03-07T03:15:07.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:07 smithi189 bash[23614]: cluster 2024-03-07T03:15:06.561509+0000 mon.a (mon.0) 63 : cluster 0 mgrmap e37: y(active, since 4s), standbys: x 2024-03-07T03:15:07.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:07 smithi189 bash[23614]: cluster 2024-03-07T03:15:07.233093+0000 mon.a (mon.0) 64 : cluster 0 osdmap e287: 8 total, 8 up, 8 in 2024-03-07T03:15:08.806 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.CloneV2ParentAbort (4060 ms) 2024-03-07T03:15:08.806 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach 2024-03-07T03:15:08.848 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:15:08.864 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 0~10 2024-03-07T03:15:08.879 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:15:09.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:09 smithi189 bash[23614]: cluster 2024-03-07T03:15:08.516213+0000 mgr.y (mgr.34107) 9 : cluster 0 pgmap v11: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:09.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:09 smithi189 bash[23614]: cluster 2024-03-07T03:15:08.570526+0000 mon.a (mon.0) 65 : cluster 0 osdmap e288: 8 total, 8 up, 8 in 2024-03-07T03:15:09.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:09 smithi189 bash[23614]: audit 2024-03-07T03:15:09.068270+0000 mon.a (mon.0) 66 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:09.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:09 smithi189 bash[23614]: audit 2024-03-07T03:15:09.076569+0000 mon.a (mon.0) 67 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:09 smithi145 bash[79878]: cluster 2024-03-07T03:15:08.516213+0000 mgr.y (mgr.34107) 9 : cluster 0 pgmap v11: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:10.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:09 smithi145 bash[79878]: cluster 2024-03-07T03:15:08.570526+0000 mon.a (mon.0) 65 : cluster 0 osdmap e288: 8 total, 8 up, 8 in 2024-03-07T03:15:10.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:09 smithi145 bash[79878]: audit 2024-03-07T03:15:09.068270+0000 mon.a (mon.0) 66 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:09 smithi145 bash[79878]: audit 2024-03-07T03:15:09.076569+0000 mon.a (mon.0) 67 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:09 smithi145 bash[30906]: cluster 2024-03-07T03:15:08.516213+0000 mgr.y (mgr.34107) 9 : cluster 0 pgmap v11: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:10.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:09 smithi145 bash[30906]: cluster 2024-03-07T03:15:08.570526+0000 mon.a (mon.0) 65 : cluster 0 osdmap e288: 8 total, 8 up, 8 in 2024-03-07T03:15:10.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:09 smithi145 bash[30906]: audit 2024-03-07T03:15:09.068270+0000 mon.a (mon.0) 66 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:09 smithi145 bash[30906]: audit 2024-03-07T03:15:09.076569+0000 mon.a (mon.0) 67 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.622 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap1 2024-03-07T03:15:10.723 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:15:10.829 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:15:10.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:10 smithi189 bash[23614]: cluster 2024-03-07T03:15:09.571287+0000 mon.a (mon.0) 68 : cluster 0 osdmap e289: 8 total, 8 up, 8 in 2024-03-07T03:15:10.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:10 smithi189 bash[23614]: audit 2024-03-07T03:15:09.603150+0000 mon.a (mon.0) 69 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:10 smithi189 bash[23614]: audit 2024-03-07T03:15:09.613259+0000 mon.a (mon.0) 70 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:10 smithi189 bash[23614]: audit 2024-03-07T03:15:10.130071+0000 mon.a (mon.0) 71 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:10 smithi189 bash[23614]: audit 2024-03-07T03:15:10.140846+0000 mon.a (mon.0) 72 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.847 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:10 smithi145 bash[79878]: cluster 2024-03-07T03:15:09.571287+0000 mon.a (mon.0) 68 : cluster 0 osdmap e289: 8 total, 8 up, 8 in 2024-03-07T03:15:10.847 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:10 smithi145 bash[79878]: audit 2024-03-07T03:15:09.603150+0000 mon.a (mon.0) 69 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.847 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:10 smithi145 bash[79878]: audit 2024-03-07T03:15:09.613259+0000 mon.a (mon.0) 70 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.847 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:10 smithi145 bash[79878]: audit 2024-03-07T03:15:10.130071+0000 mon.a (mon.0) 71 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.847 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:10 smithi145 bash[79878]: audit 2024-03-07T03:15:10.140846+0000 mon.a (mon.0) 72 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.847 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:10 smithi145 bash[30906]: cluster 2024-03-07T03:15:09.571287+0000 mon.a (mon.0) 68 : cluster 0 osdmap e289: 8 total, 8 up, 8 in 2024-03-07T03:15:10.847 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:10 smithi145 bash[30906]: audit 2024-03-07T03:15:09.603150+0000 mon.a (mon.0) 69 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.848 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:10 smithi145 bash[30906]: audit 2024-03-07T03:15:09.613259+0000 mon.a (mon.0) 70 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.848 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:10 smithi145 bash[30906]: audit 2024-03-07T03:15:10.130071+0000 mon.a (mon.0) 71 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:10.848 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:10 smithi145 bash[30906]: audit 2024-03-07T03:15:10.140846+0000 mon.a (mon.0) 72 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:12.053 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:11 smithi145 bash[30906]: cluster 2024-03-07T03:15:10.517005+0000 mgr.y (mgr.34107) 10 : cluster 0 pgmap v14: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 413 KiB/s rd, 25 KiB/s wr, 490 op/s 2024-03-07T03:15:12.053 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:11 smithi145 bash[30906]: cluster 2024-03-07T03:15:10.608328+0000 mon.a (mon.0) 73 : cluster 0 osdmap e290: 8 total, 8 up, 8 in 2024-03-07T03:15:12.053 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:11 smithi145 bash[30906]: audit 2024-03-07T03:15:10.758014+0000 mon.a (mon.0) 74 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:12.053 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:11 smithi145 bash[30906]: audit 2024-03-07T03:15:10.767015+0000 mon.a (mon.0) 75 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:12.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:11 smithi145 bash[79878]: cluster 2024-03-07T03:15:10.517005+0000 mgr.y (mgr.34107) 10 : cluster 0 pgmap v14: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 413 KiB/s rd, 25 KiB/s wr, 490 op/s 2024-03-07T03:15:12.054 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:11 smithi145 bash[79878]: cluster 2024-03-07T03:15:10.608328+0000 mon.a (mon.0) 73 : cluster 0 osdmap e290: 8 total, 8 up, 8 in 2024-03-07T03:15:12.054 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:11 smithi145 bash[79878]: audit 2024-03-07T03:15:10.758014+0000 mon.a (mon.0) 74 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:12.054 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:11 smithi145 bash[79878]: audit 2024-03-07T03:15:10.767015+0000 mon.a (mon.0) 75 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:12.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:11 smithi189 bash[23614]: cluster 2024-03-07T03:15:10.517005+0000 mgr.y (mgr.34107) 10 : cluster 0 pgmap v14: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 413 KiB/s rd, 25 KiB/s wr, 490 op/s 2024-03-07T03:15:12.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:11 smithi189 bash[23614]: cluster 2024-03-07T03:15:10.608328+0000 mon.a (mon.0) 73 : cluster 0 osdmap e290: 8 total, 8 up, 8 in 2024-03-07T03:15:12.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:11 smithi189 bash[23614]: audit 2024-03-07T03:15:10.758014+0000 mon.a (mon.0) 74 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:12.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:11 smithi189 bash[23614]: audit 2024-03-07T03:15:10.767015+0000 mon.a (mon.0) 75 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:12.130 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:15:12.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:15:12 smithi189 bash[59814]: ts=2024-03-07T03:15:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:15:13.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:15:12] "GET /metrics HTTP/1.1" 200 35883 "" "Prometheus/2.43.0" 2024-03-07T03:15:13.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:12 smithi145 bash[79878]: cluster 2024-03-07T03:15:11.772713+0000 mon.a (mon.0) 76 : cluster 0 osdmap e291: 8 total, 8 up, 8 in 2024-03-07T03:15:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:12 smithi145 bash[79878]: cluster 2024-03-07T03:15:12.232443+0000 mon.a (mon.0) 77 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2024-03-07T03:15:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:12 smithi145 bash[30906]: cluster 2024-03-07T03:15:11.772713+0000 mon.a (mon.0) 76 : cluster 0 osdmap e291: 8 total, 8 up, 8 in 2024-03-07T03:15:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:12 smithi145 bash[30906]: cluster 2024-03-07T03:15:12.232443+0000 mon.a (mon.0) 77 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2024-03-07T03:15:13.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:12 smithi189 bash[23614]: cluster 2024-03-07T03:15:11.772713+0000 mon.a (mon.0) 76 : cluster 0 osdmap e291: 8 total, 8 up, 8 in 2024-03-07T03:15:13.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:12 smithi189 bash[23614]: cluster 2024-03-07T03:15:12.232443+0000 mon.a (mon.0) 77 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2024-03-07T03:15:13.467 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach (4662 ms) 2024-03-07T03:15:13.467 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixParentReattach 2024-03-07T03:15:13.526 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:15:13.554 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 0~10 2024-03-07T03:15:13.574 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:15:13.760 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:15:13 smithi189 bash[59814]: ts=2024-03-07T03:15:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:15:14.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:13 smithi145 bash[79878]: cluster 2024-03-07T03:15:12.517798+0000 mgr.y (mgr.34107) 11 : cluster 0 pgmap v18: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:14.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:13 smithi145 bash[79878]: cluster 2024-03-07T03:15:13.246919+0000 mon.a (mon.0) 78 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2024-03-07T03:15:14.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:13 smithi145 bash[30906]: cluster 2024-03-07T03:15:12.517798+0000 mgr.y (mgr.34107) 11 : cluster 0 pgmap v18: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:14.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:13 smithi145 bash[30906]: cluster 2024-03-07T03:15:13.246919+0000 mon.a (mon.0) 78 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2024-03-07T03:15:14.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:13 smithi189 bash[23614]: cluster 2024-03-07T03:15:12.517798+0000 mgr.y (mgr.34107) 11 : cluster 0 pgmap v18: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:14.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:13 smithi189 bash[23614]: cluster 2024-03-07T03:15:13.246919+0000 mon.a (mon.0) 78 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2024-03-07T03:15:14.664 INFO:teuthology.orchestra.run.smithi145.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:15:15.283 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap1 2024-03-07T03:15:15.425 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:15:15.466 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'sleep 60' 2024-03-07T03:15:15.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:15 smithi145 bash[79878]: cluster 2024-03-07T03:15:14.248044+0000 mon.a (mon.0) 79 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2024-03-07T03:15:15.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:15 smithi145 bash[79878]: cluster 2024-03-07T03:15:14.518583+0000 mgr.y (mgr.34107) 12 : cluster 0 pgmap v21: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 315 KiB/s rd, 20 KiB/s wr, 407 op/s 2024-03-07T03:15:15.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:15 smithi145 bash[79878]: audit 2024-03-07T03:15:14.670186+0000 mgr.y (mgr.34107) 13 : audit 0 from='client.25319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:15:15.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:15 smithi145 bash[79878]: cephadm 2024-03-07T03:15:14.670646+0000 mgr.y (mgr.34107) 14 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:15:15.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:15 smithi145 bash[79878]: audit 2024-03-07T03:15:14.677031+0000 mon.a (mon.0) 80 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:15.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:15 smithi145 bash[30906]: cluster 2024-03-07T03:15:14.248044+0000 mon.a (mon.0) 79 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2024-03-07T03:15:15.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:15 smithi145 bash[30906]: cluster 2024-03-07T03:15:14.518583+0000 mgr.y (mgr.34107) 12 : cluster 0 pgmap v21: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 315 KiB/s rd, 20 KiB/s wr, 407 op/s 2024-03-07T03:15:15.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:15 smithi145 bash[30906]: audit 2024-03-07T03:15:14.670186+0000 mgr.y (mgr.34107) 13 : audit 0 from='client.25319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:15:15.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:15 smithi145 bash[30906]: cephadm 2024-03-07T03:15:14.670646+0000 mgr.y (mgr.34107) 14 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:15:15.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:15 smithi145 bash[30906]: audit 2024-03-07T03:15:14.677031+0000 mon.a (mon.0) 80 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:15.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:15 smithi189 bash[23614]: cluster 2024-03-07T03:15:14.248044+0000 mon.a (mon.0) 79 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2024-03-07T03:15:15.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:15 smithi189 bash[23614]: cluster 2024-03-07T03:15:14.518583+0000 mgr.y (mgr.34107) 12 : cluster 0 pgmap v21: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 315 KiB/s rd, 20 KiB/s wr, 407 op/s 2024-03-07T03:15:15.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:15 smithi189 bash[23614]: audit 2024-03-07T03:15:14.670186+0000 mgr.y (mgr.34107) 13 : audit 0 from='client.25319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:15:15.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:15 smithi189 bash[23614]: cephadm 2024-03-07T03:15:14.670646+0000 mgr.y (mgr.34107) 14 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:15:15.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:15 smithi189 bash[23614]: audit 2024-03-07T03:15:14.677031+0000 mon.a (mon.0) 80 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:16.434 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:16 smithi189 bash[23614]: cluster 2024-03-07T03:15:15.263982+0000 mon.a (mon.0) 81 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2024-03-07T03:15:16.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:16 smithi145 bash[79878]: cluster 2024-03-07T03:15:15.263982+0000 mon.a (mon.0) 81 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2024-03-07T03:15:16.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:16 smithi145 bash[30906]: cluster 2024-03-07T03:15:15.263982+0000 mon.a (mon.0) 81 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2024-03-07T03:15:16.658 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:15:17.491 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixParentReattach (4024 ms) 2024-03-07T03:15:17.492 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.CloneV1ParentAbortRelinkNotNeeded 2024-03-07T03:15:17.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:17 smithi145 bash[30906]: cluster 2024-03-07T03:15:16.267057+0000 mon.a (mon.0) 82 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2024-03-07T03:15:17.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:17 smithi145 bash[30906]: cluster 2024-03-07T03:15:16.519292+0000 mgr.y (mgr.34107) 15 : cluster 0 pgmap v24: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 308 KiB/s rd, 20 KiB/s wr, 398 op/s 2024-03-07T03:15:17.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:17 smithi145 bash[30906]: cluster 2024-03-07T03:15:17.237084+0000 mon.a (mon.0) 83 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2024-03-07T03:15:17.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:17 smithi145 bash[79878]: cluster 2024-03-07T03:15:16.267057+0000 mon.a (mon.0) 82 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2024-03-07T03:15:17.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:17 smithi145 bash[79878]: cluster 2024-03-07T03:15:16.519292+0000 mgr.y (mgr.34107) 15 : cluster 0 pgmap v24: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 308 KiB/s rd, 20 KiB/s wr, 398 op/s 2024-03-07T03:15:17.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:17 smithi145 bash[79878]: cluster 2024-03-07T03:15:17.237084+0000 mon.a (mon.0) 83 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2024-03-07T03:15:17.571 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:15:17.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:17 smithi189 bash[23614]: cluster 2024-03-07T03:15:16.267057+0000 mon.a (mon.0) 82 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2024-03-07T03:15:17.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:17 smithi189 bash[23614]: cluster 2024-03-07T03:15:16.519292+0000 mgr.y (mgr.34107) 15 : cluster 0 pgmap v24: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 308 KiB/s rd, 20 KiB/s wr, 398 op/s 2024-03-07T03:15:17.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:17 smithi189 bash[23614]: cluster 2024-03-07T03:15:17.237084+0000 mon.a (mon.0) 83 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2024-03-07T03:15:17.599 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 0~10 2024-03-07T03:15:17.621 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:15:18.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:18 smithi145 bash[79878]: audit 2024-03-07T03:15:17.625428+0000 mon.a (mon.0) 84 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:18.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:18 smithi145 bash[30906]: audit 2024-03-07T03:15:17.625428+0000 mon.a (mon.0) 84 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:18.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:18 smithi189 bash[23614]: audit 2024-03-07T03:15:17.625428+0000 mon.a (mon.0) 84 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:19.325 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap1 2024-03-07T03:15:19.483 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:15:19.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:19 smithi145 bash[30906]: cluster 2024-03-07T03:15:18.286726+0000 mon.a (mon.0) 85 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2024-03-07T03:15:19.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:19 smithi145 bash[30906]: cluster 2024-03-07T03:15:18.520093+0000 mgr.y (mgr.34107) 16 : cluster 0 pgmap v27: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:19.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:19 smithi145 bash[79878]: cluster 2024-03-07T03:15:18.286726+0000 mon.a (mon.0) 85 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2024-03-07T03:15:19.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:19 smithi145 bash[79878]: cluster 2024-03-07T03:15:18.520093+0000 mgr.y (mgr.34107) 16 : cluster 0 pgmap v27: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:19.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:19 smithi189 bash[23614]: cluster 2024-03-07T03:15:18.286726+0000 mon.a (mon.0) 85 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2024-03-07T03:15:19.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:19 smithi189 bash[23614]: cluster 2024-03-07T03:15:18.520093+0000 mgr.y (mgr.34107) 16 : cluster 0 pgmap v27: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:20.211 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:15:20.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: cluster 2024-03-07T03:15:19.310222+0000 mon.a (mon.0) 86 : cluster 0 osdmap e299: 8 total, 8 up, 8 in 2024-03-07T03:15:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: audit 2024-03-07T03:15:19.484459+0000 mon.a (mon.0) 87 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: audit 2024-03-07T03:15:19.499114+0000 mon.a (mon.0) 88 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: audit 2024-03-07T03:15:19.503209+0000 mon.a (mon.0) 89 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: audit 2024-03-07T03:15:19.505132+0000 mon.a (mon.0) 90 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:15:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: cephadm 2024-03-07T03:15:19.508012+0000 mgr.y (mgr.34107) 17 : cephadm 1 Updating smithi145:/etc/ceph/ceph.conf 2024-03-07T03:15:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: cephadm 2024-03-07T03:15:19.508487+0000 mgr.y (mgr.34107) 18 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T03:15:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: cephadm 2024-03-07T03:15:19.723626+0000 mgr.y (mgr.34107) 19 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:15:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: cephadm 2024-03-07T03:15:19.724259+0000 mgr.y (mgr.34107) 20 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:15:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: audit 2024-03-07T03:15:20.226499+0000 mon.a (mon.0) 91 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: audit 2024-03-07T03:15:20.232160+0000 mon.a (mon.0) 92 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: audit 2024-03-07T03:15:20.237365+0000 mon.a (mon.0) 93 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: audit 2024-03-07T03:15:20.242146+0000 mon.a (mon.0) 94 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[30906]: audit 2024-03-07T03:15:20.248584+0000 mon.a (mon.0) 95 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: cluster 2024-03-07T03:15:19.310222+0000 mon.a (mon.0) 86 : cluster 0 osdmap e299: 8 total, 8 up, 8 in 2024-03-07T03:15:20.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: audit 2024-03-07T03:15:19.484459+0000 mon.a (mon.0) 87 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: audit 2024-03-07T03:15:19.499114+0000 mon.a (mon.0) 88 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: audit 2024-03-07T03:15:19.503209+0000 mon.a (mon.0) 89 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:20.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: audit 2024-03-07T03:15:19.505132+0000 mon.a (mon.0) 90 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:15:20.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: cephadm 2024-03-07T03:15:19.508012+0000 mgr.y (mgr.34107) 17 : cephadm 1 Updating smithi145:/etc/ceph/ceph.conf 2024-03-07T03:15:20.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: cephadm 2024-03-07T03:15:19.508487+0000 mgr.y (mgr.34107) 18 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T03:15:20.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: cephadm 2024-03-07T03:15:19.723626+0000 mgr.y (mgr.34107) 19 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:15:20.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: cephadm 2024-03-07T03:15:19.724259+0000 mgr.y (mgr.34107) 20 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:15:20.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: audit 2024-03-07T03:15:20.226499+0000 mon.a (mon.0) 91 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: audit 2024-03-07T03:15:20.232160+0000 mon.a (mon.0) 92 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: audit 2024-03-07T03:15:20.237365+0000 mon.a (mon.0) 93 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: audit 2024-03-07T03:15:20.242146+0000 mon.a (mon.0) 94 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:20 smithi145 bash[79878]: audit 2024-03-07T03:15:20.248584+0000 mon.a (mon.0) 95 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: cluster 2024-03-07T03:15:19.310222+0000 mon.a (mon.0) 86 : cluster 0 osdmap e299: 8 total, 8 up, 8 in 2024-03-07T03:15:20.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: audit 2024-03-07T03:15:19.484459+0000 mon.a (mon.0) 87 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: audit 2024-03-07T03:15:19.499114+0000 mon.a (mon.0) 88 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: audit 2024-03-07T03:15:19.503209+0000 mon.a (mon.0) 89 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:20.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: audit 2024-03-07T03:15:19.505132+0000 mon.a (mon.0) 90 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:15:20.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: cephadm 2024-03-07T03:15:19.508012+0000 mgr.y (mgr.34107) 17 : cephadm 1 Updating smithi145:/etc/ceph/ceph.conf 2024-03-07T03:15:20.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: cephadm 2024-03-07T03:15:19.508487+0000 mgr.y (mgr.34107) 18 : cephadm 1 Updating smithi189:/etc/ceph/ceph.conf 2024-03-07T03:15:20.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: cephadm 2024-03-07T03:15:19.723626+0000 mgr.y (mgr.34107) 19 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:15:20.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: cephadm 2024-03-07T03:15:19.724259+0000 mgr.y (mgr.34107) 20 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:15:20.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: audit 2024-03-07T03:15:20.226499+0000 mon.a (mon.0) 91 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: audit 2024-03-07T03:15:20.232160+0000 mon.a (mon.0) 92 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: audit 2024-03-07T03:15:20.237365+0000 mon.a (mon.0) 93 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: audit 2024-03-07T03:15:20.242146+0000 mon.a (mon.0) 94 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:20 smithi189 bash[23614]: audit 2024-03-07T03:15:20.248584+0000 mon.a (mon.0) 95 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:20.921 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:15:21.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[79878]: cephadm 2024-03-07T03:15:19.887763+0000 mgr.y (mgr.34107) 21 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:15:21.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[79878]: cephadm 2024-03-07T03:15:19.888722+0000 mgr.y (mgr.34107) 22 : cephadm 1 Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:15:21.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[79878]: cephadm 2024-03-07T03:15:20.052373+0000 mgr.y (mgr.34107) 23 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:15:21.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[79878]: cephadm 2024-03-07T03:15:20.053318+0000 mgr.y (mgr.34107) 24 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:15:21.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[79878]: audit 2024-03-07T03:15:20.313473+0000 mon.a (mon.0) 96 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:21.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[79878]: audit 2024-03-07T03:15:20.316025+0000 mon.a (mon.0) 97 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:21.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[79878]: audit 2024-03-07T03:15:20.317012+0000 mon.a (mon.0) 98 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:15:21.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[79878]: audit 2024-03-07T03:15:20.322975+0000 mon.a (mon.0) 99 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:21.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[79878]: cluster 2024-03-07T03:15:20.503588+0000 mon.a (mon.0) 100 : cluster 0 osdmap e300: 8 total, 8 up, 8 in 2024-03-07T03:15:21.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[79878]: cluster 2024-03-07T03:15:20.520997+0000 mgr.y (mgr.34107) 25 : cluster 0 pgmap v30: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 286 KiB/s rd, 20 KiB/s wr, 370 op/s 2024-03-07T03:15:21.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[30906]: cephadm 2024-03-07T03:15:19.887763+0000 mgr.y (mgr.34107) 21 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:15:21.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[30906]: cephadm 2024-03-07T03:15:19.888722+0000 mgr.y (mgr.34107) 22 : cephadm 1 Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:15:21.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[30906]: cephadm 2024-03-07T03:15:20.052373+0000 mgr.y (mgr.34107) 23 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:15:21.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[30906]: cephadm 2024-03-07T03:15:20.053318+0000 mgr.y (mgr.34107) 24 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:15:21.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[30906]: audit 2024-03-07T03:15:20.313473+0000 mon.a (mon.0) 96 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:21.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[30906]: audit 2024-03-07T03:15:20.316025+0000 mon.a (mon.0) 97 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:21.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[30906]: audit 2024-03-07T03:15:20.317012+0000 mon.a (mon.0) 98 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:15:21.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[30906]: audit 2024-03-07T03:15:20.322975+0000 mon.a (mon.0) 99 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:21.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[30906]: cluster 2024-03-07T03:15:20.503588+0000 mon.a (mon.0) 100 : cluster 0 osdmap e300: 8 total, 8 up, 8 in 2024-03-07T03:15:21.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:21 smithi145 bash[30906]: cluster 2024-03-07T03:15:20.520997+0000 mgr.y (mgr.34107) 25 : cluster 0 pgmap v30: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 286 KiB/s rd, 20 KiB/s wr, 370 op/s 2024-03-07T03:15:21.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:21 smithi189 bash[23614]: cephadm 2024-03-07T03:15:19.887763+0000 mgr.y (mgr.34107) 21 : cephadm 1 Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:15:21.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:21 smithi189 bash[23614]: cephadm 2024-03-07T03:15:19.888722+0000 mgr.y (mgr.34107) 22 : cephadm 1 Updating smithi145:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:15:21.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:21 smithi189 bash[23614]: cephadm 2024-03-07T03:15:20.052373+0000 mgr.y (mgr.34107) 23 : cephadm 1 Updating smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:15:21.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:21 smithi189 bash[23614]: cephadm 2024-03-07T03:15:20.053318+0000 mgr.y (mgr.34107) 24 : cephadm 1 Updating smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:15:21.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:21 smithi189 bash[23614]: audit 2024-03-07T03:15:20.313473+0000 mon.a (mon.0) 96 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:21.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:21 smithi189 bash[23614]: audit 2024-03-07T03:15:20.316025+0000 mon.a (mon.0) 97 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:21.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:21 smithi189 bash[23614]: audit 2024-03-07T03:15:20.317012+0000 mon.a (mon.0) 98 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:15:21.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:21 smithi189 bash[23614]: audit 2024-03-07T03:15:20.322975+0000 mon.a (mon.0) 99 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:15:21.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:21 smithi189 bash[23614]: cluster 2024-03-07T03:15:20.503588+0000 mon.a (mon.0) 100 : cluster 0 osdmap e300: 8 total, 8 up, 8 in 2024-03-07T03:15:21.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:21 smithi189 bash[23614]: cluster 2024-03-07T03:15:20.520997+0000 mgr.y (mgr.34107) 25 : cluster 0 pgmap v30: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 286 KiB/s rd, 20 KiB/s wr, 370 op/s 2024-03-07T03:15:21.722 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.CloneV1ParentAbortRelinkNotNeeded (4231 ms) 2024-03-07T03:15:21.722 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach 2024-03-07T03:15:21.773 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:15:21.794 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 0~10 2024-03-07T03:15:21.811 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:15:22.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:22 smithi189 bash[23614]: cluster 2024-03-07T03:15:21.516053+0000 mon.a (mon.0) 101 : cluster 0 osdmap e301: 8 total, 8 up, 8 in 2024-03-07T03:15:22.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:22 smithi189 bash[23614]: cluster 2024-03-07T03:15:22.242136+0000 mon.a (mon.0) 102 : cluster 0 osdmap e302: 8 total, 8 up, 8 in 2024-03-07T03:15:22.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:15:22 smithi189 bash[59814]: ts=2024-03-07T03:15:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:15:22.752 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:22 smithi145 bash[79878]: cluster 2024-03-07T03:15:21.516053+0000 mon.a (mon.0) 101 : cluster 0 osdmap e301: 8 total, 8 up, 8 in 2024-03-07T03:15:22.752 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:22 smithi145 bash[79878]: cluster 2024-03-07T03:15:22.242136+0000 mon.a (mon.0) 102 : cluster 0 osdmap e302: 8 total, 8 up, 8 in 2024-03-07T03:15:22.753 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:22 smithi145 bash[30906]: cluster 2024-03-07T03:15:21.516053+0000 mon.a (mon.0) 101 : cluster 0 osdmap e301: 8 total, 8 up, 8 in 2024-03-07T03:15:22.753 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:22 smithi145 bash[30906]: cluster 2024-03-07T03:15:22.242136+0000 mon.a (mon.0) 102 : cluster 0 osdmap e302: 8 total, 8 up, 8 in 2024-03-07T03:15:23.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:15:22] "GET /metrics HTTP/1.1" 200 38699 "" "Prometheus/2.43.0" 2024-03-07T03:15:23.252 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap1 2024-03-07T03:15:23.382 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:15:23.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:23 smithi145 bash[79878]: cluster 2024-03-07T03:15:22.521971+0000 mgr.y (mgr.34107) 26 : cluster 0 pgmap v33: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 286 KiB/s rd, 20 KiB/s wr, 370 op/s 2024-03-07T03:15:23.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:23 smithi145 bash[79878]: cluster 2024-03-07T03:15:23.242004+0000 mon.a (mon.0) 103 : cluster 0 osdmap e303: 8 total, 8 up, 8 in 2024-03-07T03:15:23.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:23 smithi145 bash[30906]: cluster 2024-03-07T03:15:22.521971+0000 mgr.y (mgr.34107) 26 : cluster 0 pgmap v33: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 286 KiB/s rd, 20 KiB/s wr, 370 op/s 2024-03-07T03:15:23.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:23 smithi145 bash[30906]: cluster 2024-03-07T03:15:23.242004+0000 mon.a (mon.0) 103 : cluster 0 osdmap e303: 8 total, 8 up, 8 in 2024-03-07T03:15:23.830 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:15:23 smithi189 bash[59814]: ts=2024-03-07T03:15:23.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:15:23.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:23 smithi189 bash[23614]: cluster 2024-03-07T03:15:22.521971+0000 mgr.y (mgr.34107) 26 : cluster 0 pgmap v33: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 286 KiB/s rd, 20 KiB/s wr, 370 op/s 2024-03-07T03:15:23.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:23 smithi189 bash[23614]: cluster 2024-03-07T03:15:23.242004+0000 mon.a (mon.0) 103 : cluster 0 osdmap e303: 8 total, 8 up, 8 in 2024-03-07T03:15:24.922 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:15:25.802 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach (4079 ms) 2024-03-07T03:15:25.802 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixParentReattach 2024-03-07T03:15:25.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:25 smithi145 bash[79878]: cluster 2024-03-07T03:15:24.522757+0000 mgr.y (mgr.34107) 27 : cluster 0 pgmap v36: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 343 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-03-07T03:15:25.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:25 smithi145 bash[79878]: cluster 2024-03-07T03:15:24.524091+0000 mon.a (mon.0) 104 : cluster 0 osdmap e304: 8 total, 8 up, 8 in 2024-03-07T03:15:25.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:25 smithi145 bash[30906]: cluster 2024-03-07T03:15:24.522757+0000 mgr.y (mgr.34107) 27 : cluster 0 pgmap v36: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 343 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-03-07T03:15:25.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:25 smithi145 bash[30906]: cluster 2024-03-07T03:15:24.524091+0000 mon.a (mon.0) 104 : cluster 0 osdmap e304: 8 total, 8 up, 8 in 2024-03-07T03:15:25.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:25 smithi189 bash[23614]: cluster 2024-03-07T03:15:24.522757+0000 mgr.y (mgr.34107) 27 : cluster 0 pgmap v36: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 343 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-03-07T03:15:25.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:25 smithi189 bash[23614]: cluster 2024-03-07T03:15:24.524091+0000 mon.a (mon.0) 104 : cluster 0 osdmap e304: 8 total, 8 up, 8 in 2024-03-07T03:15:25.883 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:15:25.913 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 0~10 2024-03-07T03:15:25.936 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:15:26.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:26 smithi145 bash[79878]: cluster 2024-03-07T03:15:25.539364+0000 mon.a (mon.0) 105 : cluster 0 osdmap e305: 8 total, 8 up, 8 in 2024-03-07T03:15:26.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:26 smithi145 bash[30906]: cluster 2024-03-07T03:15:25.539364+0000 mon.a (mon.0) 105 : cluster 0 osdmap e305: 8 total, 8 up, 8 in 2024-03-07T03:15:26.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:26 smithi189 bash[23614]: cluster 2024-03-07T03:15:25.539364+0000 mon.a (mon.0) 105 : cluster 0 osdmap e305: 8 total, 8 up, 8 in 2024-03-07T03:15:27.262 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap1 2024-03-07T03:15:27.400 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:15:27.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:27 smithi145 bash[79878]: cluster 2024-03-07T03:15:26.523596+0000 mgr.y (mgr.34107) 28 : cluster 0 pgmap v38: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 321 KiB/s rd, 21 KiB/s wr, 412 op/s 2024-03-07T03:15:27.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:27 smithi145 bash[79878]: cluster 2024-03-07T03:15:26.538645+0000 mon.a (mon.0) 106 : cluster 0 osdmap e306: 8 total, 8 up, 8 in 2024-03-07T03:15:27.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:27 smithi145 bash[79878]: cluster 2024-03-07T03:15:27.242105+0000 mon.a (mon.0) 107 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2024-03-07T03:15:27.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:27 smithi145 bash[30906]: cluster 2024-03-07T03:15:26.523596+0000 mgr.y (mgr.34107) 28 : cluster 0 pgmap v38: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 321 KiB/s rd, 21 KiB/s wr, 412 op/s 2024-03-07T03:15:27.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:27 smithi145 bash[30906]: cluster 2024-03-07T03:15:26.538645+0000 mon.a (mon.0) 106 : cluster 0 osdmap e306: 8 total, 8 up, 8 in 2024-03-07T03:15:27.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:27 smithi145 bash[30906]: cluster 2024-03-07T03:15:27.242105+0000 mon.a (mon.0) 107 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2024-03-07T03:15:27.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:27 smithi189 bash[23614]: cluster 2024-03-07T03:15:26.523596+0000 mgr.y (mgr.34107) 28 : cluster 0 pgmap v38: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 321 KiB/s rd, 21 KiB/s wr, 412 op/s 2024-03-07T03:15:27.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:27 smithi189 bash[23614]: cluster 2024-03-07T03:15:26.538645+0000 mon.a (mon.0) 106 : cluster 0 osdmap e306: 8 total, 8 up, 8 in 2024-03-07T03:15:27.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:27 smithi189 bash[23614]: cluster 2024-03-07T03:15:27.242105+0000 mon.a (mon.0) 107 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2024-03-07T03:15:28.957 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:15:29.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:29 smithi145 bash[79878]: cluster 2024-03-07T03:15:28.524407+0000 mgr.y (mgr.34107) 29 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 2.7 KiB/s wr, 22 op/s 2024-03-07T03:15:29.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:29 smithi145 bash[79878]: cluster 2024-03-07T03:15:28.558009+0000 mon.a (mon.0) 108 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2024-03-07T03:15:29.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:29 smithi145 bash[30906]: cluster 2024-03-07T03:15:28.524407+0000 mgr.y (mgr.34107) 29 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 2.7 KiB/s wr, 22 op/s 2024-03-07T03:15:29.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:29 smithi145 bash[30906]: cluster 2024-03-07T03:15:28.558009+0000 mon.a (mon.0) 108 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2024-03-07T03:15:29.825 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixParentReattach (4023 ms) 2024-03-07T03:15:29.825 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.CloneV2ParentAbortRelinkNotNeeded 2024-03-07T03:15:29.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:29 smithi189 bash[23614]: cluster 2024-03-07T03:15:28.524407+0000 mgr.y (mgr.34107) 29 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 2.7 KiB/s wr, 22 op/s 2024-03-07T03:15:29.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:29 smithi189 bash[23614]: cluster 2024-03-07T03:15:28.558009+0000 mon.a (mon.0) 108 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2024-03-07T03:15:29.914 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:15:29.950 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 0~10 2024-03-07T03:15:29.974 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:15:30.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:30 smithi189 bash[23614]: cluster 2024-03-07T03:15:29.575580+0000 mon.a (mon.0) 109 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2024-03-07T03:15:31.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:30 smithi145 bash[79878]: cluster 2024-03-07T03:15:29.575580+0000 mon.a (mon.0) 109 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2024-03-07T03:15:31.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:30 smithi145 bash[30906]: cluster 2024-03-07T03:15:29.575580+0000 mon.a (mon.0) 109 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2024-03-07T03:15:31.594 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap1 2024-03-07T03:15:31.744 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:15:31.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:31 smithi189 bash[23614]: cluster 2024-03-07T03:15:30.525161+0000 mgr.y (mgr.34107) 30 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 20 KiB/s wr, 320 op/s 2024-03-07T03:15:31.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:31 smithi189 bash[23614]: cluster 2024-03-07T03:15:30.583176+0000 mon.a (mon.0) 110 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2024-03-07T03:15:32.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:31 smithi145 bash[79878]: cluster 2024-03-07T03:15:30.525161+0000 mgr.y (mgr.34107) 30 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 20 KiB/s wr, 320 op/s 2024-03-07T03:15:32.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:31 smithi145 bash[79878]: cluster 2024-03-07T03:15:30.583176+0000 mon.a (mon.0) 110 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2024-03-07T03:15:32.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:31 smithi145 bash[30906]: cluster 2024-03-07T03:15:30.525161+0000 mgr.y (mgr.34107) 30 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 247 KiB/s rd, 20 KiB/s wr, 320 op/s 2024-03-07T03:15:32.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:31 smithi145 bash[30906]: cluster 2024-03-07T03:15:30.583176+0000 mon.a (mon.0) 110 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2024-03-07T03:15:32.577 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:15:32 smithi189 bash[59814]: ts=2024-03-07T03:15:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:15:32.666 INFO:tasks.workunit.client.0.smithi145.stdout:migration_abort 2024-03-07T03:15:32.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:32 smithi189 bash[23614]: cluster 2024-03-07T03:15:31.583693+0000 mon.a (mon.0) 111 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2024-03-07T03:15:32.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:32 smithi189 bash[23614]: cluster 2024-03-07T03:15:32.249482+0000 mon.a (mon.0) 112 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2024-03-07T03:15:32.965 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:32 smithi145 bash[79878]: cluster 2024-03-07T03:15:31.583693+0000 mon.a (mon.0) 111 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2024-03-07T03:15:32.965 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:32 smithi145 bash[79878]: cluster 2024-03-07T03:15:32.249482+0000 mon.a (mon.0) 112 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2024-03-07T03:15:32.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:32 smithi145 bash[30906]: cluster 2024-03-07T03:15:31.583693+0000 mon.a (mon.0) 111 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2024-03-07T03:15:32.965 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:32 smithi145 bash[30906]: cluster 2024-03-07T03:15:32.249482+0000 mon.a (mon.0) 112 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2024-03-07T03:15:33.308 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:15:32] "GET /metrics HTTP/1.1" 200 38699 "" "Prometheus/2.43.0" 2024-03-07T03:15:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:33 smithi189 bash[23614]: cluster 2024-03-07T03:15:32.525928+0000 mgr.y (mgr.34107) 31 : cluster 0 pgmap v48: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:33.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:33 smithi189 bash[23614]: audit 2024-03-07T03:15:32.626097+0000 mon.a (mon.0) 113 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:33.831 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:15:33 smithi189 bash[59814]: ts=2024-03-07T03:15:33.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:15:33.871 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.CloneV2ParentAbortRelinkNotNeeded (4047 ms) 2024-03-07T03:15:33.871 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.StressNoMigrate 2024-03-07T03:15:33.961 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:15:33.990 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 20407257~7504559 2024-03-07T03:15:34.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:33 smithi145 bash[79878]: cluster 2024-03-07T03:15:32.525928+0000 mgr.y (mgr.34107) 31 : cluster 0 pgmap v48: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:34.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:33 smithi145 bash[79878]: audit 2024-03-07T03:15:32.626097+0000 mon.a (mon.0) 113 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:34.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:33 smithi145 bash[30906]: cluster 2024-03-07T03:15:32.525928+0000 mgr.y (mgr.34107) 31 : cluster 0 pgmap v48: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:34.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:33 smithi145 bash[30906]: audit 2024-03-07T03:15:32.626097+0000 mon.a (mon.0) 113 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:34.095 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 59934294~7473662 2024-03-07T03:15:34.095 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 19379982~1501576 2024-03-07T03:15:34.117 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 82628719~364663 2024-03-07T03:15:34.118 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 44772488~4534755 2024-03-07T03:15:34.176 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 5491085~4168816 2024-03-07T03:15:34.176 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 78984829~277163 2024-03-07T03:15:34.192 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 44442937~3885285 2024-03-07T03:15:34.198 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap0 2024-03-07T03:15:35.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:34 smithi145 bash[79878]: cluster 2024-03-07T03:15:33.605467+0000 mon.a (mon.0) 114 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2024-03-07T03:15:35.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:34 smithi145 bash[30906]: cluster 2024-03-07T03:15:33.605467+0000 mon.a (mon.0) 114 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2024-03-07T03:15:35.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:34 smithi189 bash[23614]: cluster 2024-03-07T03:15:33.605467+0000 mon.a (mon.0) 114 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2024-03-07T03:15:35.629 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 5389866~6922127 2024-03-07T03:15:35.669 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 18241960~69449 2024-03-07T03:15:35.670 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 10011430~7375973 2024-03-07T03:15:35.717 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 77595334~5147524 2024-03-07T03:15:35.717 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 29603542~5460630 2024-03-07T03:15:35.723 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 61602947~6186309 2024-03-07T03:15:35.724 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 38856693~2238445 2024-03-07T03:15:35.725 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 9712107~6512733 2024-03-07T03:15:35.748 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:15:36.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:35 smithi145 bash[79878]: cluster 2024-03-07T03:15:34.526693+0000 mgr.y (mgr.34107) 32 : cluster 0 pgmap v50: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 361 KiB/s rd, 27 KiB/s wr, 462 op/s 2024-03-07T03:15:36.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:35 smithi145 bash[79878]: cluster 2024-03-07T03:15:34.614496+0000 mon.a (mon.0) 115 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2024-03-07T03:15:36.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:35 smithi145 bash[30906]: cluster 2024-03-07T03:15:34.526693+0000 mgr.y (mgr.34107) 32 : cluster 0 pgmap v50: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 361 KiB/s rd, 27 KiB/s wr, 462 op/s 2024-03-07T03:15:36.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:35 smithi145 bash[30906]: cluster 2024-03-07T03:15:34.614496+0000 mon.a (mon.0) 115 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2024-03-07T03:15:36.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:35 smithi189 bash[23614]: cluster 2024-03-07T03:15:34.526693+0000 mgr.y (mgr.34107) 32 : cluster 0 pgmap v50: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 361 KiB/s rd, 27 KiB/s wr, 462 op/s 2024-03-07T03:15:36.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:35 smithi189 bash[23614]: cluster 2024-03-07T03:15:34.614496+0000 mon.a (mon.0) 115 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2024-03-07T03:15:37.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:36 smithi145 bash[79878]: cluster 2024-03-07T03:15:35.616406+0000 mon.a (mon.0) 116 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2024-03-07T03:15:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:36 smithi145 bash[30906]: cluster 2024-03-07T03:15:35.616406+0000 mon.a (mon.0) 116 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2024-03-07T03:15:37.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:36 smithi189 bash[23614]: cluster 2024-03-07T03:15:35.616406+0000 mon.a (mon.0) 116 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2024-03-07T03:15:37.262 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap1 2024-03-07T03:15:37.294 INFO:tasks.workunit.client.0.smithi145.stdout:clone image159 -> image162 2024-03-07T03:15:37.548 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 112978170 2024-03-07T03:15:37.577 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 2656948~6880962 2024-03-07T03:15:37.673 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 22561361~72413 2024-03-07T03:15:37.750 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 74816666~2537156 2024-03-07T03:15:37.787 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 72423567~1598966 2024-03-07T03:15:37.792 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 111368805~1060175 2024-03-07T03:15:37.814 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 90976976~4288314 2024-03-07T03:15:37.815 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 97316792~2632382 2024-03-07T03:15:37.851 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 45330356~3407813 2024-03-07T03:15:37.877 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:15:38.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:37 smithi145 bash[79878]: cluster 2024-03-07T03:15:36.527469+0000 mgr.y (mgr.34107) 33 : cluster 0 pgmap v53: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 337 KiB/s rd, 26 KiB/s wr, 432 op/s 2024-03-07T03:15:38.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:37 smithi145 bash[79878]: cluster 2024-03-07T03:15:36.637024+0000 mon.a (mon.0) 117 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2024-03-07T03:15:38.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:37 smithi145 bash[79878]: cluster 2024-03-07T03:15:37.246678+0000 mon.a (mon.0) 118 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2024-03-07T03:15:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:37 smithi145 bash[30906]: cluster 2024-03-07T03:15:36.527469+0000 mgr.y (mgr.34107) 33 : cluster 0 pgmap v53: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 337 KiB/s rd, 26 KiB/s wr, 432 op/s 2024-03-07T03:15:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:37 smithi145 bash[30906]: cluster 2024-03-07T03:15:36.637024+0000 mon.a (mon.0) 117 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2024-03-07T03:15:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:37 smithi145 bash[30906]: cluster 2024-03-07T03:15:37.246678+0000 mon.a (mon.0) 118 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2024-03-07T03:15:38.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:37 smithi189 bash[23614]: cluster 2024-03-07T03:15:36.527469+0000 mgr.y (mgr.34107) 33 : cluster 0 pgmap v53: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 337 KiB/s rd, 26 KiB/s wr, 432 op/s 2024-03-07T03:15:38.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:37 smithi189 bash[23614]: cluster 2024-03-07T03:15:36.637024+0000 mon.a (mon.0) 117 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2024-03-07T03:15:38.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:37 smithi189 bash[23614]: cluster 2024-03-07T03:15:37.246678+0000 mon.a (mon.0) 118 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2024-03-07T03:15:39.667 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap2 2024-03-07T03:15:39.704 INFO:tasks.workunit.client.0.smithi145.stdout:clone image162 -> image164 2024-03-07T03:15:40.036 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 118832631 2024-03-07T03:15:40.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:39 smithi145 bash[30906]: cluster 2024-03-07T03:15:38.528412+0000 mgr.y (mgr.34107) 34 : cluster 0 pgmap v56: 137 pgs: 137 active+clean; 940 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.8 MiB/s wr, 130 op/s 2024-03-07T03:15:40.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:39 smithi145 bash[30906]: cluster 2024-03-07T03:15:38.652907+0000 mon.a (mon.0) 119 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2024-03-07T03:15:40.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:39 smithi145 bash[79878]: cluster 2024-03-07T03:15:38.528412+0000 mgr.y (mgr.34107) 34 : cluster 0 pgmap v56: 137 pgs: 137 active+clean; 940 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.8 MiB/s wr, 130 op/s 2024-03-07T03:15:40.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:39 smithi145 bash[79878]: cluster 2024-03-07T03:15:38.652907+0000 mon.a (mon.0) 119 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2024-03-07T03:15:40.066 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 115438600~2374800 2024-03-07T03:15:40.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:39 smithi189 bash[23614]: cluster 2024-03-07T03:15:38.528412+0000 mgr.y (mgr.34107) 34 : cluster 0 pgmap v56: 137 pgs: 137 active+clean; 940 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.8 MiB/s wr, 130 op/s 2024-03-07T03:15:40.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:39 smithi189 bash[23614]: cluster 2024-03-07T03:15:38.652907+0000 mon.a (mon.0) 119 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2024-03-07T03:15:40.114 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 64497128~5293761 2024-03-07T03:15:40.133 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 46726902~4985349 2024-03-07T03:15:40.196 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 82855091~5474763 2024-03-07T03:15:40.218 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 95913036~467097 2024-03-07T03:15:40.236 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 61721500~2735175 2024-03-07T03:15:40.251 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 68248273~2094809 2024-03-07T03:15:40.280 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 91514943~4465776 2024-03-07T03:15:40.287 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:15:41.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:40 smithi145 bash[79878]: cluster 2024-03-07T03:15:39.651251+0000 mon.a (mon.0) 120 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2024-03-07T03:15:41.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:40 smithi145 bash[30906]: cluster 2024-03-07T03:15:39.651251+0000 mon.a (mon.0) 120 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2024-03-07T03:15:41.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:40 smithi189 bash[23614]: cluster 2024-03-07T03:15:39.651251+0000 mon.a (mon.0) 120 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2024-03-07T03:15:41.687 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 86189208 2024-03-07T03:15:42.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:41 smithi145 bash[79878]: cluster 2024-03-07T03:15:40.529243+0000 mgr.y (mgr.34107) 35 : cluster 0 pgmap v59: 137 pgs: 137 active+clean; 1.0 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 26 MiB/s wr, 349 op/s 2024-03-07T03:15:42.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:41 smithi145 bash[79878]: cluster 2024-03-07T03:15:40.671472+0000 mon.a (mon.0) 121 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2024-03-07T03:15:42.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:41 smithi145 bash[30906]: cluster 2024-03-07T03:15:40.529243+0000 mgr.y (mgr.34107) 35 : cluster 0 pgmap v59: 137 pgs: 137 active+clean; 1.0 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 26 MiB/s wr, 349 op/s 2024-03-07T03:15:42.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:41 smithi145 bash[30906]: cluster 2024-03-07T03:15:40.671472+0000 mon.a (mon.0) 121 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2024-03-07T03:15:42.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:41 smithi189 bash[23614]: cluster 2024-03-07T03:15:40.529243+0000 mgr.y (mgr.34107) 35 : cluster 0 pgmap v59: 137 pgs: 137 active+clean; 1.0 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 26 MiB/s wr, 349 op/s 2024-03-07T03:15:42.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:41 smithi189 bash[23614]: cluster 2024-03-07T03:15:40.671472+0000 mon.a (mon.0) 121 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2024-03-07T03:15:42.335 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.StressNoMigrate (8463 ms) 2024-03-07T03:15:42.335 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.Stress 2024-03-07T03:15:42.417 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:15:42.447 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 27866666~1276577 2024-03-07T03:15:42.486 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 39119323~725834 2024-03-07T03:15:42.486 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 45301256~6039206 2024-03-07T03:15:42.556 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 26224235~6854464 2024-03-07T03:15:42.562 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 65851658~7455343 2024-03-07T03:15:42.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:15:42 smithi189 bash[59814]: ts=2024-03-07T03:15:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:15:42.635 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 58643034~6089047 2024-03-07T03:15:42.641 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 41746136~1117594 2024-03-07T03:15:42.657 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 18067312~5216539 2024-03-07T03:15:42.657 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap0 2024-03-07T03:15:43.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:15:42] "GET /metrics HTTP/1.1" 200 38691 "" "Prometheus/2.43.0" 2024-03-07T03:15:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:42 smithi145 bash[79878]: cluster 2024-03-07T03:15:41.681205+0000 mon.a (mon.0) 122 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2024-03-07T03:15:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:42 smithi145 bash[79878]: cluster 2024-03-07T03:15:42.256967+0000 mon.a (mon.0) 123 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2024-03-07T03:15:43.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:42 smithi145 bash[30906]: cluster 2024-03-07T03:15:41.681205+0000 mon.a (mon.0) 122 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2024-03-07T03:15:43.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:42 smithi145 bash[30906]: cluster 2024-03-07T03:15:42.256967+0000 mon.a (mon.0) 123 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2024-03-07T03:15:43.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:42 smithi189 bash[23614]: cluster 2024-03-07T03:15:41.681205+0000 mon.a (mon.0) 122 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2024-03-07T03:15:43.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:42 smithi189 bash[23614]: cluster 2024-03-07T03:15:42.256967+0000 mon.a (mon.0) 123 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2024-03-07T03:15:43.830 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:15:43 smithi189 bash[59814]: ts=2024-03-07T03:15:43.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:15:43.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:43 smithi189 bash[23614]: cluster 2024-03-07T03:15:42.530237+0000 mgr.y (mgr.34107) 36 : cluster 0 pgmap v63: 137 pgs: 137 active+clean; 1.0 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-03-07T03:15:43.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:43 smithi189 bash[23614]: cluster 2024-03-07T03:15:43.260833+0000 mon.a (mon.0) 124 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2024-03-07T03:15:44.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:43 smithi145 bash[79878]: cluster 2024-03-07T03:15:42.530237+0000 mgr.y (mgr.34107) 36 : cluster 0 pgmap v63: 137 pgs: 137 active+clean; 1.0 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-03-07T03:15:44.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:43 smithi145 bash[79878]: cluster 2024-03-07T03:15:43.260833+0000 mon.a (mon.0) 124 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2024-03-07T03:15:44.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:43 smithi145 bash[30906]: cluster 2024-03-07T03:15:42.530237+0000 mgr.y (mgr.34107) 36 : cluster 0 pgmap v63: 137 pgs: 137 active+clean; 1.0 GiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-03-07T03:15:44.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:43 smithi145 bash[30906]: cluster 2024-03-07T03:15:43.260833+0000 mon.a (mon.0) 124 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2024-03-07T03:15:44.268 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap0 2024-03-07T03:15:44.298 INFO:tasks.workunit.client.0.smithi145.stdout:clone image165 -> image168 2024-03-07T03:15:44.557 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 107428155 2024-03-07T03:15:44.585 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 84926090~7897919 2024-03-07T03:15:44.676 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 87484452~1978706 2024-03-07T03:15:44.684 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 17333370~831990 2024-03-07T03:15:44.702 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 47240806~3348002 2024-03-07T03:15:44.746 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 44874627~7490127 2024-03-07T03:15:44.817 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 91250675~6938433 2024-03-07T03:15:44.836 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 65926194~8265251 2024-03-07T03:15:44.919 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 33854857~4842931 2024-03-07T03:15:44.936 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:15:45.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:45 smithi145 bash[79878]: cluster 2024-03-07T03:15:44.262222+0000 mon.a (mon.0) 125 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2024-03-07T03:15:45.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:45 smithi145 bash[79878]: cluster 2024-03-07T03:15:44.530988+0000 mgr.y (mgr.34107) 37 : cluster 0 pgmap v66: 137 pgs: 137 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 15 MiB/s wr, 405 op/s 2024-03-07T03:15:45.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:45 smithi145 bash[30906]: cluster 2024-03-07T03:15:44.262222+0000 mon.a (mon.0) 125 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2024-03-07T03:15:45.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:45 smithi145 bash[30906]: cluster 2024-03-07T03:15:44.530988+0000 mgr.y (mgr.34107) 37 : cluster 0 pgmap v66: 137 pgs: 137 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 15 MiB/s wr, 405 op/s 2024-03-07T03:15:45.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:45 smithi189 bash[23614]: cluster 2024-03-07T03:15:44.262222+0000 mon.a (mon.0) 125 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2024-03-07T03:15:45.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:45 smithi189 bash[23614]: cluster 2024-03-07T03:15:44.530988+0000 mgr.y (mgr.34107) 37 : cluster 0 pgmap v66: 137 pgs: 137 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 15 MiB/s wr, 405 op/s 2024-03-07T03:15:46.292 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap1 2024-03-07T03:15:46.319 INFO:tasks.workunit.client.0.smithi145.stdout:clone image168 -> image170 2024-03-07T03:15:46.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:46 smithi145 bash[79878]: cluster 2024-03-07T03:15:45.273793+0000 mon.a (mon.0) 126 : cluster 0 osdmap e325: 8 total, 8 up, 8 in 2024-03-07T03:15:46.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:46 smithi145 bash[30906]: cluster 2024-03-07T03:15:45.273793+0000 mon.a (mon.0) 126 : cluster 0 osdmap e325: 8 total, 8 up, 8 in 2024-03-07T03:15:46.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:46 smithi189 bash[23614]: cluster 2024-03-07T03:15:45.273793+0000 mon.a (mon.0) 126 : cluster 0 osdmap e325: 8 total, 8 up, 8 in 2024-03-07T03:15:46.642 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 51105486~3907391 2024-03-07T03:15:46.740 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 95739033~7267883 2024-03-07T03:15:46.761 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 82688133~6048725 2024-03-07T03:15:46.828 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 89468038~3767779 2024-03-07T03:15:46.834 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 51602300~4018592 2024-03-07T03:15:46.868 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 5675528~2251659 2024-03-07T03:15:46.882 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 97013293~3434127 2024-03-07T03:15:46.921 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 48979622~4817355 2024-03-07T03:15:46.977 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:15:47.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:47 smithi145 bash[79878]: cluster 2024-03-07T03:15:46.273197+0000 mon.a (mon.0) 127 : cluster 0 osdmap e326: 8 total, 8 up, 8 in 2024-03-07T03:15:47.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:47 smithi145 bash[79878]: cluster 2024-03-07T03:15:46.531807+0000 mgr.y (mgr.34107) 38 : cluster 0 pgmap v69: 137 pgs: 137 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 14 MiB/s wr, 392 op/s 2024-03-07T03:15:47.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:47 smithi145 bash[30906]: cluster 2024-03-07T03:15:46.273197+0000 mon.a (mon.0) 127 : cluster 0 osdmap e326: 8 total, 8 up, 8 in 2024-03-07T03:15:47.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:47 smithi145 bash[30906]: cluster 2024-03-07T03:15:46.531807+0000 mgr.y (mgr.34107) 38 : cluster 0 pgmap v69: 137 pgs: 137 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 14 MiB/s wr, 392 op/s 2024-03-07T03:15:47.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:47 smithi189 bash[23614]: cluster 2024-03-07T03:15:46.273197+0000 mon.a (mon.0) 127 : cluster 0 osdmap e326: 8 total, 8 up, 8 in 2024-03-07T03:15:47.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:47 smithi189 bash[23614]: cluster 2024-03-07T03:15:46.531807+0000 mgr.y (mgr.34107) 38 : cluster 0 pgmap v69: 137 pgs: 137 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 14 MiB/s wr, 392 op/s 2024-03-07T03:15:48.333 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap2 2024-03-07T03:15:48.360 INFO:tasks.workunit.client.0.smithi145.stdout:clone image170 -> image172 2024-03-07T03:15:48.558 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:48 smithi145 bash[30906]: cluster 2024-03-07T03:15:47.287549+0000 mon.a (mon.0) 128 : cluster 0 osdmap e327: 8 total, 8 up, 8 in 2024-03-07T03:15:48.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:48 smithi145 bash[30906]: audit 2024-03-07T03:15:47.626322+0000 mon.a (mon.0) 129 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:48.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:48 smithi145 bash[79878]: cluster 2024-03-07T03:15:47.287549+0000 mon.a (mon.0) 128 : cluster 0 osdmap e327: 8 total, 8 up, 8 in 2024-03-07T03:15:48.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:48 smithi145 bash[79878]: audit 2024-03-07T03:15:47.626322+0000 mon.a (mon.0) 129 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:48.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:48 smithi189 bash[23614]: cluster 2024-03-07T03:15:47.287549+0000 mon.a (mon.0) 128 : cluster 0 osdmap e327: 8 total, 8 up, 8 in 2024-03-07T03:15:48.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:48 smithi189 bash[23614]: audit 2024-03-07T03:15:47.626322+0000 mon.a (mon.0) 129 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:48.756 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 150199072 2024-03-07T03:15:48.785 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 63236488~7353459 2024-03-07T03:15:48.890 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 90635477~199259 2024-03-07T03:15:48.916 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 87772549~1108329 2024-03-07T03:15:48.999 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 34687669~5830168 2024-03-07T03:15:49.032 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 51193111~770680 2024-03-07T03:15:49.065 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 41300500~1260264 2024-03-07T03:15:49.122 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 76287526~6790486 2024-03-07T03:15:49.189 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 35402367~7851103 2024-03-07T03:15:49.197 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:15:49.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:49 smithi145 bash[79878]: cluster 2024-03-07T03:15:48.312049+0000 mon.a (mon.0) 130 : cluster 0 osdmap e328: 8 total, 8 up, 8 in 2024-03-07T03:15:49.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:49 smithi145 bash[79878]: cluster 2024-03-07T03:15:48.533365+0000 mgr.y (mgr.34107) 39 : cluster 0 pgmap v72: 137 pgs: 137 active+clean; 1.2 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 MiB/s rd, 19 MiB/s wr, 352 op/s 2024-03-07T03:15:49.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:49 smithi145 bash[30906]: cluster 2024-03-07T03:15:48.312049+0000 mon.a (mon.0) 130 : cluster 0 osdmap e328: 8 total, 8 up, 8 in 2024-03-07T03:15:49.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:49 smithi145 bash[30906]: cluster 2024-03-07T03:15:48.533365+0000 mgr.y (mgr.34107) 39 : cluster 0 pgmap v72: 137 pgs: 137 active+clean; 1.2 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 MiB/s rd, 19 MiB/s wr, 352 op/s 2024-03-07T03:15:49.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:49 smithi189 bash[23614]: cluster 2024-03-07T03:15:48.312049+0000 mon.a (mon.0) 130 : cluster 0 osdmap e328: 8 total, 8 up, 8 in 2024-03-07T03:15:49.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:49 smithi189 bash[23614]: cluster 2024-03-07T03:15:48.533365+0000 mgr.y (mgr.34107) 39 : cluster 0 pgmap v72: 137 pgs: 137 active+clean; 1.2 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 MiB/s rd, 19 MiB/s wr, 352 op/s 2024-03-07T03:15:50.355 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap3 2024-03-07T03:15:50.385 INFO:tasks.workunit.client.0.smithi145.stdout:clone image172 -> image174 2024-03-07T03:15:50.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:50 smithi145 bash[79878]: cluster 2024-03-07T03:15:49.324187+0000 mon.a (mon.0) 131 : cluster 0 osdmap e329: 8 total, 8 up, 8 in 2024-03-07T03:15:50.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:50 smithi145 bash[30906]: cluster 2024-03-07T03:15:49.324187+0000 mon.a (mon.0) 131 : cluster 0 osdmap e329: 8 total, 8 up, 8 in 2024-03-07T03:15:50.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:50 smithi189 bash[23614]: cluster 2024-03-07T03:15:49.324187+0000 mon.a (mon.0) 131 : cluster 0 osdmap e329: 8 total, 8 up, 8 in 2024-03-07T03:15:50.874 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:15:51.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:51 smithi145 bash[79878]: cluster 2024-03-07T03:15:50.340305+0000 mon.a (mon.0) 132 : cluster 0 osdmap e330: 8 total, 8 up, 8 in 2024-03-07T03:15:51.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:51 smithi145 bash[79878]: cluster 2024-03-07T03:15:50.534249+0000 mgr.y (mgr.34107) 40 : cluster 0 pgmap v75: 137 pgs: 137 active+clean; 1.2 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 24 MiB/s wr, 497 op/s 2024-03-07T03:15:51.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:51 smithi145 bash[30906]: cluster 2024-03-07T03:15:50.340305+0000 mon.a (mon.0) 132 : cluster 0 osdmap e330: 8 total, 8 up, 8 in 2024-03-07T03:15:51.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:51 smithi145 bash[30906]: cluster 2024-03-07T03:15:50.534249+0000 mgr.y (mgr.34107) 40 : cluster 0 pgmap v75: 137 pgs: 137 active+clean; 1.2 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 24 MiB/s wr, 497 op/s 2024-03-07T03:15:51.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:51 smithi189 bash[23614]: cluster 2024-03-07T03:15:50.340305+0000 mon.a (mon.0) 132 : cluster 0 osdmap e330: 8 total, 8 up, 8 in 2024-03-07T03:15:51.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:51 smithi189 bash[23614]: cluster 2024-03-07T03:15:50.534249+0000 mgr.y (mgr.34107) 40 : cluster 0 pgmap v75: 137 pgs: 137 active+clean; 1.2 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 24 MiB/s wr, 497 op/s 2024-03-07T03:15:52.079 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:15:52.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:15:52 smithi189 bash[59814]: ts=2024-03-07T03:15:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:15:52.738 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:15:53.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:15:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:15:52] "GET /metrics HTTP/1.1" 200 38701 "" "Prometheus/2.43.0" 2024-03-07T03:15:53.462 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.Stress (11127 ms) 2024-03-07T03:15:53.462 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.Stress2 2024-03-07T03:15:53.548 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:15:53.578 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 55063528~4819843 2024-03-07T03:15:53.649 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 60209367~2283140 2024-03-07T03:15:53.654 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 19050477~6983539 2024-03-07T03:15:53.713 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 75853374~4923238 2024-03-07T03:15:53.713 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 10336072~7090617 2024-03-07T03:15:53.784 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 12739962~4361359 2024-03-07T03:15:53.790 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 36880642~3045204 2024-03-07T03:15:53.825 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 61964720~6909858 2024-03-07T03:15:53.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:53 smithi189 bash[23614]: cluster 2024-03-07T03:15:52.535025+0000 mgr.y (mgr.34107) 41 : cluster 0 pgmap v76: 137 pgs: 137 active+clean; 1.2 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s rd, 18 MiB/s wr, 379 op/s 2024-03-07T03:15:53.830 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:15:53 smithi189 bash[59814]: ts=2024-03-07T03:15:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:15:53.831 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap0 2024-03-07T03:15:54.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:53 smithi145 bash[79878]: cluster 2024-03-07T03:15:52.535025+0000 mgr.y (mgr.34107) 41 : cluster 0 pgmap v76: 137 pgs: 137 active+clean; 1.2 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s rd, 18 MiB/s wr, 379 op/s 2024-03-07T03:15:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:53 smithi145 bash[30906]: cluster 2024-03-07T03:15:52.535025+0000 mgr.y (mgr.34107) 41 : cluster 0 pgmap v76: 137 pgs: 137 active+clean; 1.2 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 MiB/s rd, 18 MiB/s wr, 379 op/s 2024-03-07T03:15:55.630 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap0 2024-03-07T03:15:55.670 INFO:tasks.workunit.client.0.smithi145.stdout:clone image175 -> image178 2024-03-07T03:15:55.924 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 23498511~7560821 2024-03-07T03:15:56.040 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 67051757~1988577 2024-03-07T03:15:56.056 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 3248841~1215783 2024-03-07T03:15:56.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:55 smithi145 bash[79878]: cluster 2024-03-07T03:15:54.536738+0000 mgr.y (mgr.34107) 42 : cluster 0 pgmap v77: 137 pgs: 137 active+clean; 1.3 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 24 MiB/s wr, 1.09k op/s 2024-03-07T03:15:56.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:55 smithi145 bash[79878]: cluster 2024-03-07T03:15:54.609268+0000 mon.a (mon.0) 133 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2024-03-07T03:15:56.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:55 smithi145 bash[30906]: cluster 2024-03-07T03:15:54.536738+0000 mgr.y (mgr.34107) 42 : cluster 0 pgmap v77: 137 pgs: 137 active+clean; 1.3 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 24 MiB/s wr, 1.09k op/s 2024-03-07T03:15:56.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:55 smithi145 bash[30906]: cluster 2024-03-07T03:15:54.609268+0000 mon.a (mon.0) 133 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2024-03-07T03:15:56.079 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 66726525~526114 2024-03-07T03:15:56.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:55 smithi189 bash[23614]: cluster 2024-03-07T03:15:54.536738+0000 mgr.y (mgr.34107) 42 : cluster 0 pgmap v77: 137 pgs: 137 active+clean; 1.3 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 24 MiB/s wr, 1.09k op/s 2024-03-07T03:15:56.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:55 smithi189 bash[23614]: cluster 2024-03-07T03:15:54.609268+0000 mon.a (mon.0) 133 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2024-03-07T03:15:56.086 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 25154433~4468783 2024-03-07T03:15:56.139 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 3348842~669725 2024-03-07T03:15:56.145 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 72598432~6641766 2024-03-07T03:15:56.210 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 536298~8080723 2024-03-07T03:15:56.260 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:15:57.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:56 smithi145 bash[79878]: cluster 2024-03-07T03:15:55.619792+0000 mon.a (mon.0) 134 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2024-03-07T03:15:57.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:56 smithi145 bash[30906]: cluster 2024-03-07T03:15:55.619792+0000 mon.a (mon.0) 134 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2024-03-07T03:15:57.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:56 smithi189 bash[23614]: cluster 2024-03-07T03:15:55.619792+0000 mon.a (mon.0) 134 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2024-03-07T03:15:57.641 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap1 2024-03-07T03:15:57.667 INFO:tasks.workunit.client.0.smithi145.stdout:clone image178 -> image180 2024-03-07T03:15:58.002 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 53844957~2809560 2024-03-07T03:15:58.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:57 smithi145 bash[79878]: cluster 2024-03-07T03:15:56.537561+0000 mgr.y (mgr.34107) 43 : cluster 0 pgmap v80: 137 pgs: 137 active+clean; 1.3 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 8.6 MiB/s wr, 774 op/s 2024-03-07T03:15:58.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:57 smithi145 bash[79878]: cluster 2024-03-07T03:15:56.629165+0000 mon.a (mon.0) 135 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2024-03-07T03:15:58.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:57 smithi145 bash[30906]: cluster 2024-03-07T03:15:56.537561+0000 mgr.y (mgr.34107) 43 : cluster 0 pgmap v80: 137 pgs: 137 active+clean; 1.3 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 8.6 MiB/s wr, 774 op/s 2024-03-07T03:15:58.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:57 smithi145 bash[30906]: cluster 2024-03-07T03:15:56.629165+0000 mon.a (mon.0) 135 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2024-03-07T03:15:58.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:57 smithi189 bash[23614]: cluster 2024-03-07T03:15:56.537561+0000 mgr.y (mgr.34107) 43 : cluster 0 pgmap v80: 137 pgs: 137 active+clean; 1.3 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 8.6 MiB/s wr, 774 op/s 2024-03-07T03:15:58.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:57 smithi189 bash[23614]: cluster 2024-03-07T03:15:56.629165+0000 mon.a (mon.0) 135 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2024-03-07T03:15:58.102 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 26677393~7031556 2024-03-07T03:15:58.181 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 44947347~5460773 2024-03-07T03:15:58.250 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 16894586~5862923 2024-03-07T03:15:58.313 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 37938606~6330517 2024-03-07T03:15:58.383 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 57553292~3361834 2024-03-07T03:15:58.404 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 36088479~4013754 2024-03-07T03:15:58.439 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 47945371~5554332 2024-03-07T03:15:58.445 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:15:59.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:58 smithi145 bash[79878]: cluster 2024-03-07T03:15:57.630677+0000 mon.a (mon.0) 136 : cluster 0 osdmap e334: 8 total, 8 up, 8 in 2024-03-07T03:15:59.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:58 smithi145 bash[30906]: cluster 2024-03-07T03:15:57.630677+0000 mon.a (mon.0) 136 : cluster 0 osdmap e334: 8 total, 8 up, 8 in 2024-03-07T03:15:59.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:58 smithi189 bash[23614]: cluster 2024-03-07T03:15:57.630677+0000 mon.a (mon.0) 136 : cluster 0 osdmap e334: 8 total, 8 up, 8 in 2024-03-07T03:15:59.673 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 84748645 2024-03-07T03:15:59.701 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 5288169~3003054 2024-03-07T03:15:59.737 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 8792577~5331348 2024-03-07T03:15:59.737 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 54699014~7057501 2024-03-07T03:15:59.752 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 68679630~1244822 2024-03-07T03:15:59.753 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 51875726~8372306 2024-03-07T03:15:59.823 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 56991000~2748532 2024-03-07T03:15:59.860 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 22564974~4590680 2024-03-07T03:15:59.863 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 45220552~5302301 2024-03-07T03:15:59.863 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:16:00.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:59 smithi145 bash[79878]: cluster 2024-03-07T03:15:58.538919+0000 mgr.y (mgr.34107) 44 : cluster 0 pgmap v83: 137 pgs: 137 active+clean; 1.4 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 33 MiB/s wr, 405 op/s 2024-03-07T03:16:00.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:15:59 smithi145 bash[79878]: cluster 2024-03-07T03:15:58.653976+0000 mon.a (mon.0) 137 : cluster 0 osdmap e335: 8 total, 8 up, 8 in 2024-03-07T03:16:00.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:59 smithi145 bash[30906]: cluster 2024-03-07T03:15:58.538919+0000 mgr.y (mgr.34107) 44 : cluster 0 pgmap v83: 137 pgs: 137 active+clean; 1.4 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 33 MiB/s wr, 405 op/s 2024-03-07T03:16:00.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:15:59 smithi145 bash[30906]: cluster 2024-03-07T03:15:58.653976+0000 mon.a (mon.0) 137 : cluster 0 osdmap e335: 8 total, 8 up, 8 in 2024-03-07T03:16:00.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:59 smithi189 bash[23614]: cluster 2024-03-07T03:15:58.538919+0000 mgr.y (mgr.34107) 44 : cluster 0 pgmap v83: 137 pgs: 137 active+clean; 1.4 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 33 MiB/s wr, 405 op/s 2024-03-07T03:16:00.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:15:59 smithi189 bash[23614]: cluster 2024-03-07T03:15:58.653976+0000 mon.a (mon.0) 137 : cluster 0 osdmap e335: 8 total, 8 up, 8 in 2024-03-07T03:16:01.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:00 smithi145 bash[79878]: cluster 2024-03-07T03:15:59.653886+0000 mon.a (mon.0) 138 : cluster 0 osdmap e336: 8 total, 8 up, 8 in 2024-03-07T03:16:01.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:00 smithi145 bash[30906]: cluster 2024-03-07T03:15:59.653886+0000 mon.a (mon.0) 138 : cluster 0 osdmap e336: 8 total, 8 up, 8 in 2024-03-07T03:16:01.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:00 smithi189 bash[23614]: cluster 2024-03-07T03:15:59.653886+0000 mon.a (mon.0) 138 : cluster 0 osdmap e336: 8 total, 8 up, 8 in 2024-03-07T03:16:01.692 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap3 2024-03-07T03:16:01.727 INFO:tasks.workunit.client.0.smithi145.stdout:clone image180 -> image182 2024-03-07T03:16:02.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:01 smithi145 bash[79878]: cluster 2024-03-07T03:16:00.539703+0000 mgr.y (mgr.34107) 45 : cluster 0 pgmap v86: 137 pgs: 137 active+clean; 1.5 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 36 MiB/s wr, 461 op/s 2024-03-07T03:16:02.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:01 smithi145 bash[79878]: cluster 2024-03-07T03:16:00.676280+0000 mon.a (mon.0) 139 : cluster 0 osdmap e337: 8 total, 8 up, 8 in 2024-03-07T03:16:02.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:01 smithi145 bash[30906]: cluster 2024-03-07T03:16:00.539703+0000 mgr.y (mgr.34107) 45 : cluster 0 pgmap v86: 137 pgs: 137 active+clean; 1.5 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 36 MiB/s wr, 461 op/s 2024-03-07T03:16:02.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:01 smithi145 bash[30906]: cluster 2024-03-07T03:16:00.676280+0000 mon.a (mon.0) 139 : cluster 0 osdmap e337: 8 total, 8 up, 8 in 2024-03-07T03:16:02.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:01 smithi189 bash[23614]: cluster 2024-03-07T03:16:00.539703+0000 mgr.y (mgr.34107) 45 : cluster 0 pgmap v86: 137 pgs: 137 active+clean; 1.5 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 36 MiB/s wr, 461 op/s 2024-03-07T03:16:02.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:01 smithi189 bash[23614]: cluster 2024-03-07T03:16:00.676280+0000 mon.a (mon.0) 139 : cluster 0 osdmap e337: 8 total, 8 up, 8 in 2024-03-07T03:16:02.150 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:16:02.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:16:02 smithi189 bash[59814]: ts=2024-03-07T03:16:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:03.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:16:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:16:02] "GET /metrics HTTP/1.1" 200 38694 "" "Prometheus/2.43.0" 2024-03-07T03:16:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:02 smithi145 bash[79878]: cluster 2024-03-07T03:16:01.674748+0000 mon.a (mon.0) 140 : cluster 0 osdmap e338: 8 total, 8 up, 8 in 2024-03-07T03:16:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:02 smithi145 bash[79878]: audit 2024-03-07T03:16:02.630394+0000 mon.a (mon.0) 141 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:02 smithi145 bash[30906]: cluster 2024-03-07T03:16:01.674748+0000 mon.a (mon.0) 140 : cluster 0 osdmap e338: 8 total, 8 up, 8 in 2024-03-07T03:16:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:02 smithi145 bash[30906]: audit 2024-03-07T03:16:02.630394+0000 mon.a (mon.0) 141 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:02 smithi189 bash[23614]: cluster 2024-03-07T03:16:01.674748+0000 mon.a (mon.0) 140 : cluster 0 osdmap e338: 8 total, 8 up, 8 in 2024-03-07T03:16:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:02 smithi189 bash[23614]: audit 2024-03-07T03:16:02.630394+0000 mon.a (mon.0) 141 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:03.148 INFO:tasks.workunit.client.0.smithi145.stdout:write: a 10740378~3576565 2024-03-07T03:16:03.226 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 27088608~6248518 2024-03-07T03:16:03.274 INFO:tasks.workunit.client.0.smithi145.stdout:write: a 37773421~6813211 2024-03-07T03:16:03.352 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 26749997~3585885 2024-03-07T03:16:03.359 INFO:tasks.workunit.client.0.smithi145.stdout:write: a 34279956~2247278 2024-03-07T03:16:03.415 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 15680871~3435591 2024-03-07T03:16:03.447 INFO:tasks.workunit.client.0.smithi145.stdout:write: a 27054602~2062173 2024-03-07T03:16:03.464 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 15545088~7016562 2024-03-07T03:16:03.506 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: user0 2024-03-07T03:16:03.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:03 smithi189 bash[23614]: cluster 2024-03-07T03:16:02.540452+0000 mgr.y (mgr.34107) 46 : cluster 0 pgmap v89: 137 pgs: 137 active+clean; 1.5 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.3 MiB/s wr, 55 op/s 2024-03-07T03:16:03.830 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:16:03 smithi189 bash[59814]: ts=2024-03-07T03:16:03.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:03 smithi145 bash[79878]: cluster 2024-03-07T03:16:02.540452+0000 mgr.y (mgr.34107) 46 : cluster 0 pgmap v89: 137 pgs: 137 active+clean; 1.5 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.3 MiB/s wr, 55 op/s 2024-03-07T03:16:04.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:03 smithi145 bash[30906]: cluster 2024-03-07T03:16:02.540452+0000 mgr.y (mgr.34107) 46 : cluster 0 pgmap v89: 137 pgs: 137 active+clean; 1.5 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.3 MiB/s wr, 55 op/s 2024-03-07T03:16:04.731 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 150405402 2024-03-07T03:16:04.762 INFO:tasks.workunit.client.0.smithi145.stdout:write: b 99867089~7685105 2024-03-07T03:16:04.806 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 107017995~1455309 2024-03-07T03:16:04.843 INFO:tasks.workunit.client.0.smithi145.stdout:write: b 125771147~4067644 2024-03-07T03:16:04.860 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 38540484~5939547 2024-03-07T03:16:04.860 INFO:tasks.workunit.client.0.smithi145.stdout:write: b 101612890~2077075 2024-03-07T03:16:04.867 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 38830495~6371966 2024-03-07T03:16:04.918 INFO:tasks.workunit.client.0.smithi145.stdout:write: b 90162339~2771241 2024-03-07T03:16:04.920 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 26386517~473852 2024-03-07T03:16:04.920 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: user1 2024-03-07T03:16:05.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:04 smithi145 bash[79878]: cluster 2024-03-07T03:16:03.695721+0000 mon.a (mon.0) 142 : cluster 0 osdmap e339: 8 total, 8 up, 8 in 2024-03-07T03:16:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:04 smithi145 bash[30906]: cluster 2024-03-07T03:16:03.695721+0000 mon.a (mon.0) 142 : cluster 0 osdmap e339: 8 total, 8 up, 8 in 2024-03-07T03:16:05.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:04 smithi189 bash[23614]: cluster 2024-03-07T03:16:03.695721+0000 mon.a (mon.0) 142 : cluster 0 osdmap e339: 8 total, 8 up, 8 in 2024-03-07T03:16:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:05 smithi145 bash[79878]: cluster 2024-03-07T03:16:04.541347+0000 mgr.y (mgr.34107) 47 : cluster 0 pgmap v91: 137 pgs: 137 active+clean; 1.6 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 26 MiB/s wr, 545 op/s 2024-03-07T03:16:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:05 smithi145 bash[79878]: cluster 2024-03-07T03:16:04.721015+0000 mon.a (mon.0) 143 : cluster 0 osdmap e340: 8 total, 8 up, 8 in 2024-03-07T03:16:06.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:05 smithi145 bash[30906]: cluster 2024-03-07T03:16:04.541347+0000 mgr.y (mgr.34107) 47 : cluster 0 pgmap v91: 137 pgs: 137 active+clean; 1.6 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 26 MiB/s wr, 545 op/s 2024-03-07T03:16:06.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:05 smithi145 bash[30906]: cluster 2024-03-07T03:16:04.721015+0000 mon.a (mon.0) 143 : cluster 0 osdmap e340: 8 total, 8 up, 8 in 2024-03-07T03:16:06.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:05 smithi189 bash[23614]: cluster 2024-03-07T03:16:04.541347+0000 mgr.y (mgr.34107) 47 : cluster 0 pgmap v91: 137 pgs: 137 active+clean; 1.6 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 26 MiB/s wr, 545 op/s 2024-03-07T03:16:06.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:05 smithi189 bash[23614]: cluster 2024-03-07T03:16:04.721015+0000 mon.a (mon.0) 143 : cluster 0 osdmap e340: 8 total, 8 up, 8 in 2024-03-07T03:16:06.764 INFO:tasks.workunit.client.0.smithi145.stdout:write: c 143118338~7287064 2024-03-07T03:16:06.811 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 21350658~4717939 2024-03-07T03:16:06.811 INFO:tasks.workunit.client.0.smithi145.stdout:write: c 110707637~7959984 2024-03-07T03:16:06.837 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 21693713~3292697 2024-03-07T03:16:06.887 INFO:tasks.workunit.client.0.smithi145.stdout:write: c 98333759~7979721 2024-03-07T03:16:06.903 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 48927675~7216723 2024-03-07T03:16:06.904 INFO:tasks.workunit.client.0.smithi145.stdout:write: c 136827653~2384436 2024-03-07T03:16:06.906 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 21448176~6281132 2024-03-07T03:16:06.906 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: user2 2024-03-07T03:16:07.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:06 smithi145 bash[79878]: cluster 2024-03-07T03:16:05.729908+0000 mon.a (mon.0) 144 : cluster 0 mgrmap e38: y(active, since 63s), standbys: x 2024-03-07T03:16:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:06 smithi145 bash[79878]: cluster 2024-03-07T03:16:05.746277+0000 mon.a (mon.0) 145 : cluster 0 osdmap e341: 8 total, 8 up, 8 in 2024-03-07T03:16:07.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:06 smithi145 bash[30906]: cluster 2024-03-07T03:16:05.729908+0000 mon.a (mon.0) 144 : cluster 0 mgrmap e38: y(active, since 63s), standbys: x 2024-03-07T03:16:07.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:06 smithi145 bash[30906]: cluster 2024-03-07T03:16:05.746277+0000 mon.a (mon.0) 145 : cluster 0 osdmap e341: 8 total, 8 up, 8 in 2024-03-07T03:16:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:06 smithi189 bash[23614]: cluster 2024-03-07T03:16:05.729908+0000 mon.a (mon.0) 144 : cluster 0 mgrmap e38: y(active, since 63s), standbys: x 2024-03-07T03:16:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:06 smithi189 bash[23614]: cluster 2024-03-07T03:16:05.746277+0000 mon.a (mon.0) 145 : cluster 0 osdmap e341: 8 total, 8 up, 8 in 2024-03-07T03:16:08.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:07 smithi145 bash[79878]: cluster 2024-03-07T03:16:06.542253+0000 mgr.y (mgr.34107) 48 : cluster 0 pgmap v94: 137 pgs: 137 active+clean; 1.6 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 26 MiB/s wr, 547 op/s 2024-03-07T03:16:08.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:07 smithi145 bash[79878]: cluster 2024-03-07T03:16:06.751968+0000 mon.a (mon.0) 146 : cluster 0 osdmap e342: 8 total, 8 up, 8 in 2024-03-07T03:16:08.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:07 smithi145 bash[30906]: cluster 2024-03-07T03:16:06.542253+0000 mgr.y (mgr.34107) 48 : cluster 0 pgmap v94: 137 pgs: 137 active+clean; 1.6 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 26 MiB/s wr, 547 op/s 2024-03-07T03:16:08.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:07 smithi145 bash[30906]: cluster 2024-03-07T03:16:06.751968+0000 mon.a (mon.0) 146 : cluster 0 osdmap e342: 8 total, 8 up, 8 in 2024-03-07T03:16:08.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:07 smithi189 bash[23614]: cluster 2024-03-07T03:16:06.542253+0000 mgr.y (mgr.34107) 48 : cluster 0 pgmap v94: 137 pgs: 137 active+clean; 1.6 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 MiB/s rd, 26 MiB/s wr, 547 op/s 2024-03-07T03:16:08.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:07 smithi189 bash[23614]: cluster 2024-03-07T03:16:06.751968+0000 mon.a (mon.0) 146 : cluster 0 osdmap e342: 8 total, 8 up, 8 in 2024-03-07T03:16:08.786 INFO:tasks.workunit.client.0.smithi145.stdout:write: d 76437010~5561327 2024-03-07T03:16:08.841 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 73758382~1784113 2024-03-07T03:16:08.841 INFO:tasks.workunit.client.0.smithi145.stdout:write: d 134406645~619712 2024-03-07T03:16:08.842 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 55419680~5965507 2024-03-07T03:16:08.842 INFO:tasks.workunit.client.0.smithi145.stdout:write: d 31409384~883664 2024-03-07T03:16:08.843 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 127186636~162993 2024-03-07T03:16:08.845 INFO:tasks.workunit.client.0.smithi145.stdout:write: d 20430431~979291 2024-03-07T03:16:08.849 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 56299082~255972 2024-03-07T03:16:08.944 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: user3 2024-03-07T03:16:09.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:08 smithi145 bash[79878]: cluster 2024-03-07T03:16:07.754000+0000 mon.a (mon.0) 147 : cluster 0 osdmap e343: 8 total, 8 up, 8 in 2024-03-07T03:16:09.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:08 smithi145 bash[30906]: cluster 2024-03-07T03:16:07.754000+0000 mon.a (mon.0) 147 : cluster 0 osdmap e343: 8 total, 8 up, 8 in 2024-03-07T03:16:09.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:08 smithi189 bash[23614]: cluster 2024-03-07T03:16:07.754000+0000 mon.a (mon.0) 147 : cluster 0 osdmap e343: 8 total, 8 up, 8 in 2024-03-07T03:16:10.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:09 smithi145 bash[79878]: cluster 2024-03-07T03:16:08.543752+0000 mgr.y (mgr.34107) 49 : cluster 0 pgmap v97: 137 pgs: 137 active+clean; 1.7 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 5.8 MiB/s rd, 24 MiB/s wr, 222 op/s 2024-03-07T03:16:10.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:09 smithi145 bash[79878]: cluster 2024-03-07T03:16:08.773360+0000 mon.a (mon.0) 148 : cluster 0 osdmap e344: 8 total, 8 up, 8 in 2024-03-07T03:16:10.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:09 smithi145 bash[30906]: cluster 2024-03-07T03:16:08.543752+0000 mgr.y (mgr.34107) 49 : cluster 0 pgmap v97: 137 pgs: 137 active+clean; 1.7 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 5.8 MiB/s rd, 24 MiB/s wr, 222 op/s 2024-03-07T03:16:10.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:09 smithi145 bash[30906]: cluster 2024-03-07T03:16:08.773360+0000 mon.a (mon.0) 148 : cluster 0 osdmap e344: 8 total, 8 up, 8 in 2024-03-07T03:16:10.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:09 smithi189 bash[23614]: cluster 2024-03-07T03:16:08.543752+0000 mgr.y (mgr.34107) 49 : cluster 0 pgmap v97: 137 pgs: 137 active+clean; 1.7 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 5.8 MiB/s rd, 24 MiB/s wr, 222 op/s 2024-03-07T03:16:10.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:09 smithi189 bash[23614]: cluster 2024-03-07T03:16:08.773360+0000 mon.a (mon.0) 148 : cluster 0 osdmap e344: 8 total, 8 up, 8 in 2024-03-07T03:16:10.835 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 15040540~3760135 2024-03-07T03:16:10.837 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 15980573~1880067 2024-03-07T03:16:10.928 INFO:tasks.workunit.client.0.smithi145.stdout:write: 2 30081080~3760135 2024-03-07T03:16:10.931 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 31021113~1880067 2024-03-07T03:16:10.998 INFO:tasks.workunit.client.0.smithi145.stdout:write: 3 45121620~3760135 2024-03-07T03:16:11.001 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 46061653~1880067 2024-03-07T03:16:11.052 INFO:tasks.workunit.client.0.smithi145.stdout:write: 4 60162160~3760135 2024-03-07T03:16:11.055 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 61102193~1880067 2024-03-07T03:16:11.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:10 smithi145 bash[79878]: cluster 2024-03-07T03:16:09.773537+0000 mon.a (mon.0) 149 : cluster 0 osdmap e345: 8 total, 8 up, 8 in 2024-03-07T03:16:11.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:10 smithi145 bash[30906]: cluster 2024-03-07T03:16:09.773537+0000 mon.a (mon.0) 149 : cluster 0 osdmap e345: 8 total, 8 up, 8 in 2024-03-07T03:16:11.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:10 smithi189 bash[23614]: cluster 2024-03-07T03:16:09.773537+0000 mon.a (mon.0) 149 : cluster 0 osdmap e345: 8 total, 8 up, 8 in 2024-03-07T03:16:11.106 INFO:tasks.workunit.client.0.smithi145.stdout:write: 5 75202701~3760135 2024-03-07T03:16:11.108 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 76142734~1880067 2024-03-07T03:16:12.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:11 smithi145 bash[30906]: cluster 2024-03-07T03:16:10.544487+0000 mgr.y (mgr.34107) 50 : cluster 0 pgmap v100: 137 pgs: 137 active+clean; 1.7 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 5.8 MiB/s rd, 24 MiB/s wr, 222 op/s 2024-03-07T03:16:12.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:11 smithi145 bash[30906]: cluster 2024-03-07T03:16:10.801360+0000 mon.a (mon.0) 150 : cluster 0 osdmap e346: 8 total, 8 up, 8 in 2024-03-07T03:16:12.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:11 smithi145 bash[79878]: cluster 2024-03-07T03:16:10.544487+0000 mgr.y (mgr.34107) 50 : cluster 0 pgmap v100: 137 pgs: 137 active+clean; 1.7 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 5.8 MiB/s rd, 24 MiB/s wr, 222 op/s 2024-03-07T03:16:12.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:11 smithi145 bash[79878]: cluster 2024-03-07T03:16:10.801360+0000 mon.a (mon.0) 150 : cluster 0 osdmap e346: 8 total, 8 up, 8 in 2024-03-07T03:16:12.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:11 smithi189 bash[23614]: cluster 2024-03-07T03:16:10.544487+0000 mgr.y (mgr.34107) 50 : cluster 0 pgmap v100: 137 pgs: 137 active+clean; 1.7 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 5.8 MiB/s rd, 24 MiB/s wr, 222 op/s 2024-03-07T03:16:12.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:11 smithi189 bash[23614]: cluster 2024-03-07T03:16:10.801360+0000 mon.a (mon.0) 150 : cluster 0 osdmap e346: 8 total, 8 up, 8 in 2024-03-07T03:16:12.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:16:12 smithi189 bash[59814]: ts=2024-03-07T03:16:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:12.918 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:16:13.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:16:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:16:12] "GET /metrics HTTP/1.1" 200 38694 "" "Prometheus/2.43.0" 2024-03-07T03:16:13.501 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:16:13.787 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:16:13 smithi189 bash[59814]: ts=2024-03-07T03:16:13.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:14.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:13 smithi145 bash[79878]: cluster 2024-03-07T03:16:12.545230+0000 mgr.y (mgr.34107) 51 : cluster 0 pgmap v102: 137 pgs: 137 active+clean; 1.7 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 4.8 MiB/s rd, 20 MiB/s wr, 185 op/s 2024-03-07T03:16:14.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:13 smithi145 bash[30906]: cluster 2024-03-07T03:16:12.545230+0000 mgr.y (mgr.34107) 51 : cluster 0 pgmap v102: 137 pgs: 137 active+clean; 1.7 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 4.8 MiB/s rd, 20 MiB/s wr, 185 op/s 2024-03-07T03:16:14.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:13 smithi189 bash[23614]: cluster 2024-03-07T03:16:12.545230+0000 mgr.y (mgr.34107) 51 : cluster 0 pgmap v102: 137 pgs: 137 active+clean; 1.7 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 4.8 MiB/s rd, 20 MiB/s wr, 185 op/s 2024-03-07T03:16:15.516 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.Stress2 (22054 ms) 2024-03-07T03:16:15.516 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMigration.StressLive 2024-03-07T03:16:15.601 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 83886080 2024-03-07T03:16:15.640 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 64983628~113079 2024-03-07T03:16:15.665 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 28571600~3541926 2024-03-07T03:16:15.665 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 70241903~6463534 2024-03-07T03:16:15.734 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 13680257~4748511 2024-03-07T03:16:15.735 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 65853204~1729066 2024-03-07T03:16:15.753 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 82469468~1416612 2024-03-07T03:16:15.753 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 40939924~3415300 2024-03-07T03:16:15.794 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 62485552~5182917 2024-03-07T03:16:15.817 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap0 2024-03-07T03:16:16.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:15 smithi145 bash[79878]: cluster 2024-03-07T03:16:14.546988+0000 mgr.y (mgr.34107) 52 : cluster 0 pgmap v103: 137 pgs: 137 active+clean; 1.8 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 105 MiB/s rd, 14 MiB/s wr, 448 op/s 2024-03-07T03:16:16.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:15 smithi145 bash[30906]: cluster 2024-03-07T03:16:14.546988+0000 mgr.y (mgr.34107) 52 : cluster 0 pgmap v103: 137 pgs: 137 active+clean; 1.8 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 105 MiB/s rd, 14 MiB/s wr, 448 op/s 2024-03-07T03:16:16.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:15 smithi189 bash[23614]: cluster 2024-03-07T03:16:14.546988+0000 mgr.y (mgr.34107) 52 : cluster 0 pgmap v103: 137 pgs: 137 active+clean; 1.8 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 105 MiB/s rd, 14 MiB/s wr, 448 op/s 2024-03-07T03:16:17.801 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:17 smithi145 bash[79878]: cluster 2024-03-07T03:16:16.547692+0000 mgr.y (mgr.34107) 53 : cluster 0 pgmap v104: 137 pgs: 137 active+clean; 1.8 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s rd, 10 MiB/s wr, 345 op/s 2024-03-07T03:16:17.801 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:17 smithi145 bash[79878]: cluster 2024-03-07T03:16:16.823537+0000 mon.a (mon.0) 151 : cluster 0 osdmap e347: 8 total, 8 up, 8 in 2024-03-07T03:16:17.801 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:17 smithi145 bash[79878]: audit 2024-03-07T03:16:17.630657+0000 mon.a (mon.0) 152 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:17.840 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap0 2024-03-07T03:16:17.872 INFO:tasks.workunit.client.0.smithi145.stdout:clone image183 -> image186 2024-03-07T03:16:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:17 smithi145 bash[30906]: cluster 2024-03-07T03:16:16.547692+0000 mgr.y (mgr.34107) 53 : cluster 0 pgmap v104: 137 pgs: 137 active+clean; 1.8 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s rd, 10 MiB/s wr, 345 op/s 2024-03-07T03:16:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:17 smithi145 bash[30906]: cluster 2024-03-07T03:16:16.823537+0000 mon.a (mon.0) 151 : cluster 0 osdmap e347: 8 total, 8 up, 8 in 2024-03-07T03:16:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:17 smithi145 bash[30906]: audit 2024-03-07T03:16:17.630657+0000 mon.a (mon.0) 152 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:17 smithi189 bash[23614]: cluster 2024-03-07T03:16:16.547692+0000 mgr.y (mgr.34107) 53 : cluster 0 pgmap v104: 137 pgs: 137 active+clean; 1.8 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 81 MiB/s rd, 10 MiB/s wr, 345 op/s 2024-03-07T03:16:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:17 smithi189 bash[23614]: cluster 2024-03-07T03:16:16.823537+0000 mon.a (mon.0) 151 : cluster 0 osdmap e347: 8 total, 8 up, 8 in 2024-03-07T03:16:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:17 smithi189 bash[23614]: audit 2024-03-07T03:16:17.630657+0000 mon.a (mon.0) 152 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:18.126 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 138602736 2024-03-07T03:16:18.156 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 127316194~4642187 2024-03-07T03:16:18.227 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 72253307~2988079 2024-03-07T03:16:18.270 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 122688144~6220222 2024-03-07T03:16:18.331 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 118729385~6883367 2024-03-07T03:16:18.337 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 68718972~4446120 2024-03-07T03:16:18.388 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 39628530~5502902 2024-03-07T03:16:18.415 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 55043058~7252419 2024-03-07T03:16:18.505 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 43831219~5612424 2024-03-07T03:16:18.517 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap1 2024-03-07T03:16:19.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:18 smithi189 bash[23614]: cluster 2024-03-07T03:16:17.828196+0000 mon.a (mon.0) 153 : cluster 0 osdmap e348: 8 total, 8 up, 8 in 2024-03-07T03:16:19.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:18 smithi189 bash[23614]: cluster 2024-03-07T03:16:18.549140+0000 mgr.y (mgr.34107) 54 : cluster 0 pgmap v107: 137 pgs: 137 active+clean; 1.9 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 159 MiB/s rd, 19 MiB/s wr, 600 op/s 2024-03-07T03:16:19.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:18 smithi145 bash[79878]: cluster 2024-03-07T03:16:17.828196+0000 mon.a (mon.0) 153 : cluster 0 osdmap e348: 8 total, 8 up, 8 in 2024-03-07T03:16:19.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:18 smithi145 bash[79878]: cluster 2024-03-07T03:16:18.549140+0000 mgr.y (mgr.34107) 54 : cluster 0 pgmap v107: 137 pgs: 137 active+clean; 1.9 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 159 MiB/s rd, 19 MiB/s wr, 600 op/s 2024-03-07T03:16:19.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:18 smithi145 bash[30906]: cluster 2024-03-07T03:16:17.828196+0000 mon.a (mon.0) 153 : cluster 0 osdmap e348: 8 total, 8 up, 8 in 2024-03-07T03:16:19.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:18 smithi145 bash[30906]: cluster 2024-03-07T03:16:18.549140+0000 mgr.y (mgr.34107) 54 : cluster 0 pgmap v107: 137 pgs: 137 active+clean; 1.9 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 159 MiB/s rd, 19 MiB/s wr, 600 op/s 2024-03-07T03:16:19.863 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 101770755 2024-03-07T03:16:19.920 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 9367070~6744242 2024-03-07T03:16:19.948 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 86674970~5762439 2024-03-07T03:16:19.949 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 5855362~1707511 2024-03-07T03:16:19.950 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 50577322~7157739 2024-03-07T03:16:19.951 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 62865552~4554427 2024-03-07T03:16:19.967 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 79536007~6797004 2024-03-07T03:16:19.978 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 97578439~4192316 2024-03-07T03:16:19.981 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 37735304~3538906 2024-03-07T03:16:20.012 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap2 2024-03-07T03:16:20.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:19 smithi145 bash[79878]: cluster 2024-03-07T03:16:18.840863+0000 mon.a (mon.0) 154 : cluster 0 osdmap e349: 8 total, 8 up, 8 in 2024-03-07T03:16:20.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:19 smithi145 bash[30906]: cluster 2024-03-07T03:16:18.840863+0000 mon.a (mon.0) 154 : cluster 0 osdmap e349: 8 total, 8 up, 8 in 2024-03-07T03:16:20.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:19 smithi189 bash[23614]: cluster 2024-03-07T03:16:18.840863+0000 mon.a (mon.0) 154 : cluster 0 osdmap e349: 8 total, 8 up, 8 in 2024-03-07T03:16:21.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:20 smithi145 bash[79878]: cluster 2024-03-07T03:16:19.840707+0000 mon.a (mon.0) 155 : cluster 0 osdmap e350: 8 total, 8 up, 8 in 2024-03-07T03:16:21.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:20 smithi145 bash[79878]: audit 2024-03-07T03:16:20.399922+0000 mon.a (mon.0) 156 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:21.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:20 smithi145 bash[79878]: cluster 2024-03-07T03:16:20.549852+0000 mgr.y (mgr.34107) 55 : cluster 0 pgmap v110: 137 pgs: 137 active+clean; 1.9 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 151 MiB/s rd, 17 MiB/s wr, 491 op/s 2024-03-07T03:16:21.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:20 smithi145 bash[30906]: cluster 2024-03-07T03:16:19.840707+0000 mon.a (mon.0) 155 : cluster 0 osdmap e350: 8 total, 8 up, 8 in 2024-03-07T03:16:21.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:20 smithi145 bash[30906]: audit 2024-03-07T03:16:20.399922+0000 mon.a (mon.0) 156 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:21.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:20 smithi145 bash[30906]: cluster 2024-03-07T03:16:20.549852+0000 mgr.y (mgr.34107) 55 : cluster 0 pgmap v110: 137 pgs: 137 active+clean; 1.9 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 151 MiB/s rd, 17 MiB/s wr, 491 op/s 2024-03-07T03:16:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:20 smithi189 bash[23614]: cluster 2024-03-07T03:16:19.840707+0000 mon.a (mon.0) 155 : cluster 0 osdmap e350: 8 total, 8 up, 8 in 2024-03-07T03:16:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:20 smithi189 bash[23614]: audit 2024-03-07T03:16:20.399922+0000 mon.a (mon.0) 156 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:20 smithi189 bash[23614]: cluster 2024-03-07T03:16:20.549852+0000 mgr.y (mgr.34107) 55 : cluster 0 pgmap v110: 137 pgs: 137 active+clean; 1.9 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 151 MiB/s rd, 17 MiB/s wr, 491 op/s 2024-03-07T03:16:21.892 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap2 2024-03-07T03:16:21.916 INFO:tasks.workunit.client.0.smithi145.stdout:clone image186 -> image188 2024-03-07T03:16:22.229 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 89413024 2024-03-07T03:16:22.274 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 15176067~922998 2024-03-07T03:16:22.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:21 smithi189 bash[23614]: cluster 2024-03-07T03:16:20.857321+0000 mon.a (mon.0) 157 : cluster 0 osdmap e351: 8 total, 8 up, 8 in 2024-03-07T03:16:22.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:21 smithi145 bash[30906]: cluster 2024-03-07T03:16:20.857321+0000 mon.a (mon.0) 157 : cluster 0 osdmap e351: 8 total, 8 up, 8 in 2024-03-07T03:16:22.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:21 smithi145 bash[79878]: cluster 2024-03-07T03:16:20.857321+0000 mon.a (mon.0) 157 : cluster 0 osdmap e351: 8 total, 8 up, 8 in 2024-03-07T03:16:22.355 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 82449366~6963658 2024-03-07T03:16:22.375 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 56551885~4817989 2024-03-07T03:16:22.457 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 69936206~1414959 2024-03-07T03:16:22.517 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 39676415~8097057 2024-03-07T03:16:22.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:16:22 smithi189 bash[59814]: ts=2024-03-07T03:16:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:22.606 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 31892372~2398661 2024-03-07T03:16:22.623 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 16324007~1223185 2024-03-07T03:16:22.647 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 74220392~4100379 2024-03-07T03:16:22.667 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: snap3 2024-03-07T03:16:23.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:22 smithi145 bash[79878]: cluster 2024-03-07T03:16:21.876050+0000 mon.a (mon.0) 158 : cluster 0 osdmap e352: 8 total, 8 up, 8 in 2024-03-07T03:16:23.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:22 smithi145 bash[79878]: cluster 2024-03-07T03:16:22.550711+0000 mgr.y (mgr.34107) 56 : cluster 0 pgmap v113: 137 pgs: 137 active+clean; 1.9 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-03-07T03:16:23.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:16:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:16:22] "GET /metrics HTTP/1.1" 200 38695 "" "Prometheus/2.43.0" 2024-03-07T03:16:23.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:22 smithi145 bash[30906]: cluster 2024-03-07T03:16:21.876050+0000 mon.a (mon.0) 158 : cluster 0 osdmap e352: 8 total, 8 up, 8 in 2024-03-07T03:16:23.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:22 smithi145 bash[30906]: cluster 2024-03-07T03:16:22.550711+0000 mgr.y (mgr.34107) 56 : cluster 0 pgmap v113: 137 pgs: 137 active+clean; 1.9 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-03-07T03:16:23.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:22 smithi189 bash[23614]: cluster 2024-03-07T03:16:21.876050+0000 mon.a (mon.0) 158 : cluster 0 osdmap e352: 8 total, 8 up, 8 in 2024-03-07T03:16:23.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:22 smithi189 bash[23614]: cluster 2024-03-07T03:16:22.550711+0000 mgr.y (mgr.34107) 56 : cluster 0 pgmap v113: 137 pgs: 137 active+clean; 1.9 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-03-07T03:16:23.830 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:16:23 smithi189 bash[59814]: ts=2024-03-07T03:16:23.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:23.915 INFO:tasks.workunit.client.0.smithi145.stdout:snap_protect: snap3 2024-03-07T03:16:23.941 INFO:tasks.workunit.client.0.smithi145.stdout:clone image188 -> image190 2024-03-07T03:16:24.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:23 smithi145 bash[79878]: cluster 2024-03-07T03:16:22.883474+0000 mon.a (mon.0) 159 : cluster 0 osdmap e353: 8 total, 8 up, 8 in 2024-03-07T03:16:24.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:23 smithi145 bash[30906]: cluster 2024-03-07T03:16:22.883474+0000 mon.a (mon.0) 159 : cluster 0 osdmap e353: 8 total, 8 up, 8 in 2024-03-07T03:16:24.299 INFO:tasks.workunit.client.0.smithi145.stdout:migration_prepare 2024-03-07T03:16:24.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:23 smithi189 bash[23614]: cluster 2024-03-07T03:16:22.883474+0000 mon.a (mon.0) 159 : cluster 0 osdmap e353: 8 total, 8 up, 8 in 2024-03-07T03:16:24.799 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade resume' 2024-03-07T03:16:25.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:24 smithi145 bash[30906]: cluster 2024-03-07T03:16:23.889924+0000 mon.a (mon.0) 160 : cluster 0 osdmap e354: 8 total, 8 up, 8 in 2024-03-07T03:16:25.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:24 smithi145 bash[30906]: cluster 2024-03-07T03:16:24.551476+0000 mgr.y (mgr.34107) 57 : cluster 0 pgmap v116: 137 pgs: 137 active+clean; 2.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7.7 MiB/s rd, 19 MiB/s wr, 385 op/s 2024-03-07T03:16:25.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:24 smithi145 bash[79878]: cluster 2024-03-07T03:16:23.889924+0000 mon.a (mon.0) 160 : cluster 0 osdmap e354: 8 total, 8 up, 8 in 2024-03-07T03:16:25.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:24 smithi145 bash[79878]: cluster 2024-03-07T03:16:24.551476+0000 mgr.y (mgr.34107) 57 : cluster 0 pgmap v116: 137 pgs: 137 active+clean; 2.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7.7 MiB/s rd, 19 MiB/s wr, 385 op/s 2024-03-07T03:16:25.063 INFO:tasks.workunit.client.0.smithi145.stdout:migration_execute 2024-03-07T03:16:25.063 INFO:tasks.workunit.client.0.smithi145.stdout:write: a 39230267~8153171 2024-03-07T03:16:25.157 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 19739108~1055329 2024-03-07T03:16:25.189 INFO:tasks.workunit.client.0.smithi145.stdout:write: a 77300012~3102311 2024-03-07T03:16:25.243 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 4717509~3507008 2024-03-07T03:16:25.280 INFO:tasks.workunit.client.0.smithi145.stdout:write: a 48154295~5145063 2024-03-07T03:16:25.330 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 4821122~1106274 2024-03-07T03:16:25.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:24 smithi189 bash[23614]: cluster 2024-03-07T03:16:23.889924+0000 mon.a (mon.0) 160 : cluster 0 osdmap e354: 8 total, 8 up, 8 in 2024-03-07T03:16:25.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:24 smithi189 bash[23614]: cluster 2024-03-07T03:16:24.551476+0000 mgr.y (mgr.34107) 57 : cluster 0 pgmap v116: 137 pgs: 137 active+clean; 2.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7.7 MiB/s rd, 19 MiB/s wr, 385 op/s 2024-03-07T03:16:25.333 INFO:tasks.workunit.client.0.smithi145.stdout:write: a 17515897~4660885 2024-03-07T03:16:25.400 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 963424~6193925 2024-03-07T03:16:25.444 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: user0 2024-03-07T03:16:26.922 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 120849354 2024-03-07T03:16:26.939 INFO:tasks.workunit.client.0.smithi145.stdout:write: b 5927302~674797 2024-03-07T03:16:26.945 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 75880718~4665112 2024-03-07T03:16:26.946 INFO:tasks.workunit.client.0.smithi145.stdout:write: b 21736078~4349076 2024-03-07T03:16:26.949 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 64025523~7109794 2024-03-07T03:16:26.950 INFO:tasks.workunit.client.0.smithi145.stdout:write: b 71234359~3159749 2024-03-07T03:16:26.956 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 108177221~6567693 2024-03-07T03:16:26.956 INFO:tasks.workunit.client.0.smithi145.stdout:write: b 7369192~7711075 2024-03-07T03:16:26.986 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 115192204~5657150 2024-03-07T03:16:26.986 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: user1 2024-03-07T03:16:27.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:26 smithi145 bash[79878]: cluster 2024-03-07T03:16:25.910700+0000 mon.a (mon.0) 161 : cluster 0 osdmap e355: 8 total, 8 up, 8 in 2024-03-07T03:16:27.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:26 smithi145 bash[79878]: audit 2024-03-07T03:16:26.541696+0000 mon.a (mon.0) 162 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:27.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:26 smithi145 bash[79878]: audit 2024-03-07T03:16:26.547386+0000 mon.a (mon.0) 163 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:27.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:26 smithi145 bash[79878]: cluster 2024-03-07T03:16:26.551920+0000 mgr.y (mgr.34107) 58 : cluster 0 pgmap v118: 137 pgs: 137 active+clean; 2.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 6.5 MiB/s rd, 17 MiB/s wr, 329 op/s 2024-03-07T03:16:27.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:26 smithi145 bash[30906]: cluster 2024-03-07T03:16:25.910700+0000 mon.a (mon.0) 161 : cluster 0 osdmap e355: 8 total, 8 up, 8 in 2024-03-07T03:16:27.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:26 smithi145 bash[30906]: audit 2024-03-07T03:16:26.541696+0000 mon.a (mon.0) 162 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:27.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:26 smithi145 bash[30906]: audit 2024-03-07T03:16:26.547386+0000 mon.a (mon.0) 163 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:27.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:26 smithi145 bash[30906]: cluster 2024-03-07T03:16:26.551920+0000 mgr.y (mgr.34107) 58 : cluster 0 pgmap v118: 137 pgs: 137 active+clean; 2.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 6.5 MiB/s rd, 17 MiB/s wr, 329 op/s 2024-03-07T03:16:27.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:26 smithi189 bash[23614]: cluster 2024-03-07T03:16:25.910700+0000 mon.a (mon.0) 161 : cluster 0 osdmap e355: 8 total, 8 up, 8 in 2024-03-07T03:16:27.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:26 smithi189 bash[23614]: audit 2024-03-07T03:16:26.541696+0000 mon.a (mon.0) 162 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:27.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:26 smithi189 bash[23614]: audit 2024-03-07T03:16:26.547386+0000 mon.a (mon.0) 163 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:27.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:26 smithi189 bash[23614]: cluster 2024-03-07T03:16:26.551920+0000 mgr.y (mgr.34107) 58 : cluster 0 pgmap v118: 137 pgs: 137 active+clean; 2.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 6.5 MiB/s rd, 17 MiB/s wr, 329 op/s 2024-03-07T03:16:28.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:27 smithi145 bash[30906]: cluster 2024-03-07T03:16:26.917311+0000 mon.a (mon.0) 164 : cluster 0 osdmap e356: 8 total, 8 up, 8 in 2024-03-07T03:16:28.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:27 smithi145 bash[30906]: audit 2024-03-07T03:16:27.149908+0000 mon.a (mon.0) 165 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:28.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:27 smithi145 bash[30906]: audit 2024-03-07T03:16:27.155673+0000 mon.a (mon.0) 166 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:28.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:27 smithi145 bash[30906]: audit 2024-03-07T03:16:27.672595+0000 mon.a (mon.0) 167 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:28.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:27 smithi145 bash[30906]: audit 2024-03-07T03:16:27.674235+0000 mon.a (mon.0) 168 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:28.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:27 smithi145 bash[30906]: audit 2024-03-07T03:16:27.681759+0000 mon.a (mon.0) 169 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:28.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:27 smithi145 bash[79878]: cluster 2024-03-07T03:16:26.917311+0000 mon.a (mon.0) 164 : cluster 0 osdmap e356: 8 total, 8 up, 8 in 2024-03-07T03:16:28.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:27 smithi145 bash[79878]: audit 2024-03-07T03:16:27.149908+0000 mon.a (mon.0) 165 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:28.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:27 smithi145 bash[79878]: audit 2024-03-07T03:16:27.155673+0000 mon.a (mon.0) 166 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:28.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:27 smithi145 bash[79878]: audit 2024-03-07T03:16:27.672595+0000 mon.a (mon.0) 167 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:28.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:27 smithi145 bash[79878]: audit 2024-03-07T03:16:27.674235+0000 mon.a (mon.0) 168 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:28.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:27 smithi145 bash[79878]: audit 2024-03-07T03:16:27.681759+0000 mon.a (mon.0) 169 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:28.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:27 smithi189 bash[23614]: cluster 2024-03-07T03:16:26.917311+0000 mon.a (mon.0) 164 : cluster 0 osdmap e356: 8 total, 8 up, 8 in 2024-03-07T03:16:28.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:27 smithi189 bash[23614]: audit 2024-03-07T03:16:27.149908+0000 mon.a (mon.0) 165 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:28.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:27 smithi189 bash[23614]: audit 2024-03-07T03:16:27.155673+0000 mon.a (mon.0) 166 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:28.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:27 smithi189 bash[23614]: audit 2024-03-07T03:16:27.672595+0000 mon.a (mon.0) 167 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:28.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:27 smithi189 bash[23614]: audit 2024-03-07T03:16:27.674235+0000 mon.a (mon.0) 168 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:28.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:27 smithi189 bash[23614]: audit 2024-03-07T03:16:27.681759+0000 mon.a (mon.0) 169 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:28.961 INFO:tasks.workunit.client.0.smithi145.stdout:write: c 98704419~370853 2024-03-07T03:16:28.980 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 38936169~765348 2024-03-07T03:16:28.980 INFO:tasks.workunit.client.0.smithi145.stdout:write: c 78727481~7129016 2024-03-07T03:16:29.008 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 3223778~471309 2024-03-07T03:16:29.009 INFO:tasks.workunit.client.0.smithi145.stdout:write: c 48622406~1874193 2024-03-07T03:16:29.010 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 53336865~8380767 2024-03-07T03:16:29.012 INFO:tasks.workunit.client.0.smithi145.stdout:write: c 15182857~2567148 2024-03-07T03:16:29.023 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 115064428~5784926 2024-03-07T03:16:29.023 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: user2 2024-03-07T03:16:29.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:28 smithi145 bash[30906]: cluster 2024-03-07T03:16:27.930370+0000 mon.a (mon.0) 170 : cluster 0 osdmap e357: 8 total, 8 up, 8 in 2024-03-07T03:16:29.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:28 smithi145 bash[30906]: cluster 2024-03-07T03:16:28.553490+0000 mgr.y (mgr.34107) 59 : cluster 0 pgmap v121: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 22 MiB/s rd, 29 MiB/s wr, 707 op/s 2024-03-07T03:16:29.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:28 smithi145 bash[79878]: cluster 2024-03-07T03:16:27.930370+0000 mon.a (mon.0) 170 : cluster 0 osdmap e357: 8 total, 8 up, 8 in 2024-03-07T03:16:29.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:28 smithi145 bash[79878]: cluster 2024-03-07T03:16:28.553490+0000 mgr.y (mgr.34107) 59 : cluster 0 pgmap v121: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 22 MiB/s rd, 29 MiB/s wr, 707 op/s 2024-03-07T03:16:29.329 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:16:29.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:28 smithi189 bash[23614]: cluster 2024-03-07T03:16:27.930370+0000 mon.a (mon.0) 170 : cluster 0 osdmap e357: 8 total, 8 up, 8 in 2024-03-07T03:16:29.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:28 smithi189 bash[23614]: cluster 2024-03-07T03:16:28.553490+0000 mgr.y (mgr.34107) 59 : cluster 0 pgmap v121: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 22 MiB/s rd, 29 MiB/s wr, 707 op/s 2024-03-07T03:16:30.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:29 smithi145 bash[79878]: cluster 2024-03-07T03:16:28.943624+0000 mon.a (mon.0) 171 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2024-03-07T03:16:30.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:29 smithi145 bash[30906]: cluster 2024-03-07T03:16:28.943624+0000 mon.a (mon.0) 171 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2024-03-07T03:16:30.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:29 smithi189 bash[23614]: cluster 2024-03-07T03:16:28.943624+0000 mon.a (mon.0) 171 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2024-03-07T03:16:30.976 INFO:tasks.workunit.client.0.smithi145.stdout:write: d 89911922~6168173 2024-03-07T03:16:31.019 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 3051441~5955744 2024-03-07T03:16:31.020 INFO:tasks.workunit.client.0.smithi145.stdout:write: d 5439657~5581402 2024-03-07T03:16:31.047 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 68497445~7710925 2024-03-07T03:16:31.049 INFO:tasks.workunit.client.0.smithi145.stdout:write: d 85405718~1690120 2024-03-07T03:16:31.050 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 58982849~1704136 2024-03-07T03:16:31.050 INFO:tasks.workunit.client.0.smithi145.stdout:write: d 112460158~7411189 2024-03-07T03:16:31.083 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 85929550~7718124 2024-03-07T03:16:31.099 INFO:tasks.workunit.client.0.smithi145.stdout:snap_create: user3 2024-03-07T03:16:31.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:30 smithi145 bash[79878]: cluster 2024-03-07T03:16:29.951566+0000 mon.a (mon.0) 172 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2024-03-07T03:16:31.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:30 smithi145 bash[79878]: cluster 2024-03-07T03:16:30.554266+0000 mgr.y (mgr.34107) 60 : cluster 0 pgmap v124: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s rd, 33 MiB/s wr, 824 op/s 2024-03-07T03:16:31.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:30 smithi145 bash[30906]: cluster 2024-03-07T03:16:29.951566+0000 mon.a (mon.0) 172 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2024-03-07T03:16:31.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:30 smithi145 bash[30906]: cluster 2024-03-07T03:16:30.554266+0000 mgr.y (mgr.34107) 60 : cluster 0 pgmap v124: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s rd, 33 MiB/s wr, 824 op/s 2024-03-07T03:16:31.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:30 smithi189 bash[23614]: cluster 2024-03-07T03:16:29.951566+0000 mon.a (mon.0) 172 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2024-03-07T03:16:31.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:30 smithi189 bash[23614]: cluster 2024-03-07T03:16:30.554266+0000 mgr.y (mgr.34107) 60 : cluster 0 pgmap v124: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s rd, 33 MiB/s wr, 824 op/s 2024-03-07T03:16:32.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:31 smithi189 bash[23614]: cluster 2024-03-07T03:16:30.951131+0000 mon.a (mon.0) 173 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2024-03-07T03:16:32.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:31 smithi145 bash[79878]: cluster 2024-03-07T03:16:30.951131+0000 mon.a (mon.0) 173 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2024-03-07T03:16:32.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:31 smithi145 bash[30906]: cluster 2024-03-07T03:16:30.951131+0000 mon.a (mon.0) 173 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2024-03-07T03:16:32.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:16:32 smithi189 bash[59814]: ts=2024-03-07T03:16:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:32.858 INFO:teuthology.orchestra.run.smithi145.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:16:32.991 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 191378731 2024-03-07T03:16:33.028 INFO:tasks.workunit.client.0.smithi145.stdout:write: 1 19137873~4784468 2024-03-07T03:16:33.035 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 20333990~2392234 2024-03-07T03:16:33.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:16:32] "GET /metrics HTTP/1.1" 200 38695 "" "Prometheus/2.43.0" 2024-03-07T03:16:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[30906]: cluster 2024-03-07T03:16:31.966990+0000 mon.a (mon.0) 174 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2024-03-07T03:16:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[30906]: cluster 2024-03-07T03:16:32.554718+0000 mgr.y (mgr.34107) 61 : cluster 0 pgmap v127: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-03-07T03:16:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[30906]: audit 2024-03-07T03:16:32.629708+0000 mon.a (mon.0) 175 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[30906]: audit 2024-03-07T03:16:32.870116+0000 mon.a (mon.0) 176 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[30906]: audit 2024-03-07T03:16:32.881909+0000 mon.a (mon.0) 177 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[30906]: audit 2024-03-07T03:16:32.884047+0000 mon.a (mon.0) 178 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[30906]: audit 2024-03-07T03:16:32.884986+0000 mon.a (mon.0) 179 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[30906]: audit 2024-03-07T03:16:32.892745+0000 mon.a (mon.0) 180 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[30906]: audit 2024-03-07T03:16:32.952246+0000 mon.a (mon.0) 181 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[30906]: audit 2024-03-07T03:16:32.954272+0000 mon.a (mon.0) 182 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:16:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[30906]: audit 2024-03-07T03:16:32.955562+0000 mon.a (mon.0) 183 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:16:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[30906]: audit 2024-03-07T03:16:32.956254+0000 mon.a (mon.0) 184 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-07T03:16:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[79878]: cluster 2024-03-07T03:16:31.966990+0000 mon.a (mon.0) 174 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2024-03-07T03:16:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[79878]: cluster 2024-03-07T03:16:32.554718+0000 mgr.y (mgr.34107) 61 : cluster 0 pgmap v127: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-03-07T03:16:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[79878]: audit 2024-03-07T03:16:32.629708+0000 mon.a (mon.0) 175 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[79878]: audit 2024-03-07T03:16:32.870116+0000 mon.a (mon.0) 176 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[79878]: audit 2024-03-07T03:16:32.881909+0000 mon.a (mon.0) 177 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[79878]: audit 2024-03-07T03:16:32.884047+0000 mon.a (mon.0) 178 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[79878]: audit 2024-03-07T03:16:32.884986+0000 mon.a (mon.0) 179 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[79878]: audit 2024-03-07T03:16:32.892745+0000 mon.a (mon.0) 180 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[79878]: audit 2024-03-07T03:16:32.952246+0000 mon.a (mon.0) 181 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[79878]: audit 2024-03-07T03:16:32.954272+0000 mon.a (mon.0) 182 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:16:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[79878]: audit 2024-03-07T03:16:32.955562+0000 mon.a (mon.0) 183 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:16:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:32 smithi145 bash[79878]: audit 2024-03-07T03:16:32.956254+0000 mon.a (mon.0) 184 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-07T03:16:33.065 INFO:tasks.workunit.client.0.smithi145.stdout:write: 2 38275746~4784468 2024-03-07T03:16:33.067 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 39471863~2392234 2024-03-07T03:16:33.104 INFO:tasks.workunit.client.0.smithi145.stdout:write: 3 57413619~4784468 2024-03-07T03:16:33.105 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 58609736~2392234 2024-03-07T03:16:33.131 INFO:tasks.workunit.client.0.smithi145.stdout:write: 4 76551492~4784468 2024-03-07T03:16:33.133 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 77747609~2392234 2024-03-07T03:16:33.159 INFO:tasks.workunit.client.0.smithi145.stdout:write: 5 95689365~4784468 2024-03-07T03:16:33.161 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 96885482~2392234 2024-03-07T03:16:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:32 smithi189 bash[23614]: cluster 2024-03-07T03:16:31.966990+0000 mon.a (mon.0) 174 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2024-03-07T03:16:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:32 smithi189 bash[23614]: cluster 2024-03-07T03:16:32.554718+0000 mgr.y (mgr.34107) 61 : cluster 0 pgmap v127: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-03-07T03:16:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:32 smithi189 bash[23614]: audit 2024-03-07T03:16:32.629708+0000 mon.a (mon.0) 175 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:32 smithi189 bash[23614]: audit 2024-03-07T03:16:32.870116+0000 mon.a (mon.0) 176 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:32 smithi189 bash[23614]: audit 2024-03-07T03:16:32.881909+0000 mon.a (mon.0) 177 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:32 smithi189 bash[23614]: audit 2024-03-07T03:16:32.884047+0000 mon.a (mon.0) 178 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:32 smithi189 bash[23614]: audit 2024-03-07T03:16:32.884986+0000 mon.a (mon.0) 179 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:33.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:32 smithi189 bash[23614]: audit 2024-03-07T03:16:32.892745+0000 mon.a (mon.0) 180 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:33.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:32 smithi189 bash[23614]: audit 2024-03-07T03:16:32.952246+0000 mon.a (mon.0) 181 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:33.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:32 smithi189 bash[23614]: audit 2024-03-07T03:16:32.954272+0000 mon.a (mon.0) 182 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:16:33.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:32 smithi189 bash[23614]: audit 2024-03-07T03:16:32.955562+0000 mon.a (mon.0) 183 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:16:33.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:32 smithi189 bash[23614]: audit 2024-03-07T03:16:32.956254+0000 mon.a (mon.0) 184 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-07T03:16:33.339 INFO:tasks.workunit.client.0.smithi145.stdout:migration_commit 2024-03-07T03:16:33.830 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:16:33 smithi189 bash[59814]: ts=2024-03-07T03:16:33.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:34.171 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'echo wait for majority of mons to upgrade' 2024-03-07T03:16:34.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:33 smithi145 bash[79878]: audit 2024-03-07T03:16:32.860526+0000 mgr.y (mgr.34107) 62 : audit 0 from='client.25325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:16:34.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:33 smithi145 bash[79878]: cephadm 2024-03-07T03:16:32.861221+0000 mgr.y (mgr.34107) 63 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:16:34.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:33 smithi145 bash[79878]: cephadm 2024-03-07T03:16:32.956724+0000 mgr.y (mgr.34107) 64 : cephadm 1 Upgrade: It appears safe to stop mon.c 2024-03-07T03:16:34.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:33 smithi145 bash[79878]: cluster 2024-03-07T03:16:32.987776+0000 mon.a (mon.0) 185 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2024-03-07T03:16:34.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:33 smithi145 bash[30906]: audit 2024-03-07T03:16:32.860526+0000 mgr.y (mgr.34107) 62 : audit 0 from='client.25325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:16:34.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:33 smithi145 bash[30906]: cephadm 2024-03-07T03:16:32.861221+0000 mgr.y (mgr.34107) 63 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:16:34.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:33 smithi145 bash[30906]: cephadm 2024-03-07T03:16:32.956724+0000 mgr.y (mgr.34107) 64 : cephadm 1 Upgrade: It appears safe to stop mon.c 2024-03-07T03:16:34.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:33 smithi145 bash[30906]: cluster 2024-03-07T03:16:32.987776+0000 mon.a (mon.0) 185 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2024-03-07T03:16:34.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:33 smithi189 bash[23614]: audit 2024-03-07T03:16:32.860526+0000 mgr.y (mgr.34107) 62 : audit 0 from='client.25325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:16:34.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:33 smithi189 bash[23614]: cephadm 2024-03-07T03:16:32.861221+0000 mgr.y (mgr.34107) 63 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:16:34.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:33 smithi189 bash[23614]: cephadm 2024-03-07T03:16:32.956724+0000 mgr.y (mgr.34107) 64 : cephadm 1 Upgrade: It appears safe to stop mon.c 2024-03-07T03:16:34.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:33 smithi189 bash[23614]: cluster 2024-03-07T03:16:32.987776+0000 mon.a (mon.0) 185 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2024-03-07T03:16:34.803 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMigration.StressLive (19287 ms) 2024-03-07T03:16:35.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:34 smithi145 bash[79878]: cluster 2024-03-07T03:16:34.555731+0000 mgr.y (mgr.34107) 65 : cluster 0 pgmap v129: 137 pgs: 137 active+clean; 2.3 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 26 MiB/s wr, 492 op/s 2024-03-07T03:16:35.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:34 smithi145 bash[30906]: cluster 2024-03-07T03:16:34.555731+0000 mgr.y (mgr.34107) 65 : cluster 0 pgmap v129: 137 pgs: 137 active+clean; 2.3 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 26 MiB/s wr, 492 op/s 2024-03-07T03:16:35.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:34 smithi189 bash[23614]: cluster 2024-03-07T03:16:34.555731+0000 mgr.y (mgr.34107) 65 : cluster 0 pgmap v129: 137 pgs: 137 active+clean; 2.3 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 26 MiB/s wr, 492 op/s 2024-03-07T03:16:35.997 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 57 tests from TestMigration (289499 ms total) 2024-03-07T03:16:35.997 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:16:35.997 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 6 tests from TestGroup 2024-03-07T03:16:36.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[30906]: cluster 2024-03-07T03:16:35.000908+0000 mon.a (mon.0) 186 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2024-03-07T03:16:36.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[30906]: cephadm 2024-03-07T03:16:35.725722+0000 mgr.y (mgr.34107) 66 : cephadm 1 Upgrade: Updating mon.c 2024-03-07T03:16:36.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[30906]: audit 2024-03-07T03:16:35.732680+0000 mon.a (mon.0) 187 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:36.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[30906]: audit 2024-03-07T03:16:35.736494+0000 mon.a (mon.0) 188 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:16:36.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[30906]: audit 2024-03-07T03:16:35.737662+0000 mon.a (mon.0) 189 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:16:36.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[30906]: audit 2024-03-07T03:16:35.738873+0000 mon.a (mon.0) 190 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:36.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[30906]: cephadm 2024-03-07T03:16:35.740576+0000 mgr.y (mgr.34107) 67 : cephadm 1 Deploying daemon mon.c on smithi145 2024-03-07T03:16:36.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[79878]: cluster 2024-03-07T03:16:35.000908+0000 mon.a (mon.0) 186 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2024-03-07T03:16:36.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[79878]: cephadm 2024-03-07T03:16:35.725722+0000 mgr.y (mgr.34107) 66 : cephadm 1 Upgrade: Updating mon.c 2024-03-07T03:16:36.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[79878]: audit 2024-03-07T03:16:35.732680+0000 mon.a (mon.0) 187 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:36.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[79878]: audit 2024-03-07T03:16:35.736494+0000 mon.a (mon.0) 188 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:16:36.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[79878]: audit 2024-03-07T03:16:35.737662+0000 mon.a (mon.0) 189 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:16:36.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[79878]: audit 2024-03-07T03:16:35.738873+0000 mon.a (mon.0) 190 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:36.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:36 smithi145 bash[79878]: cephadm 2024-03-07T03:16:35.740576+0000 mgr.y (mgr.34107) 67 : cephadm 1 Deploying daemon mon.c on smithi145 2024-03-07T03:16:36.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:36 smithi189 bash[23614]: cluster 2024-03-07T03:16:35.000908+0000 mon.a (mon.0) 186 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2024-03-07T03:16:36.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:36 smithi189 bash[23614]: cephadm 2024-03-07T03:16:35.725722+0000 mgr.y (mgr.34107) 66 : cephadm 1 Upgrade: Updating mon.c 2024-03-07T03:16:36.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:36 smithi189 bash[23614]: audit 2024-03-07T03:16:35.732680+0000 mon.a (mon.0) 187 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:36.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:36 smithi189 bash[23614]: audit 2024-03-07T03:16:35.736494+0000 mon.a (mon.0) 188 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:16:36.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:36 smithi189 bash[23614]: audit 2024-03-07T03:16:35.737662+0000 mon.a (mon.0) 189 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:16:36.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:36 smithi189 bash[23614]: audit 2024-03-07T03:16:35.738873+0000 mon.a (mon.0) 190 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:36.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:36 smithi189 bash[23614]: cephadm 2024-03-07T03:16:35.740576+0000 mgr.y (mgr.34107) 67 : cephadm 1 Deploying daemon mon.c on smithi145 2024-03-07T03:16:37.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:37 smithi145 bash[79878]: cluster 2024-03-07T03:16:36.004355+0000 mon.a (mon.0) 191 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2024-03-07T03:16:37.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:37 smithi145 bash[79878]: cluster 2024-03-07T03:16:36.556384+0000 mgr.y (mgr.34107) 68 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-03-07T03:16:37.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:37 smithi145 bash[30906]: cluster 2024-03-07T03:16:36.004355+0000 mon.a (mon.0) 191 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2024-03-07T03:16:37.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:37 smithi145 bash[30906]: cluster 2024-03-07T03:16:36.556384+0000 mgr.y (mgr.34107) 68 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-03-07T03:16:37.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:37 smithi189 bash[23614]: cluster 2024-03-07T03:16:36.004355+0000 mon.a (mon.0) 191 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2024-03-07T03:16:37.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:37 smithi189 bash[23614]: cluster 2024-03-07T03:16:36.556384+0000 mgr.y (mgr.34107) 68 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-03-07T03:16:38.052 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestGroup.group_create 2024-03-07T03:16:38.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:38 smithi145 bash[79878]: cluster 2024-03-07T03:16:37.018560+0000 mon.a (mon.0) 192 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:16:38.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:38 smithi145 bash[79878]: cluster 2024-03-07T03:16:37.044924+0000 mon.a (mon.0) 193 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2024-03-07T03:16:38.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:38 smithi145 bash[79878]: audit 2024-03-07T03:16:37.050279+0000 mon.c (mon.2) 183 : audit 1 from='client.? 172.21.15.145:0/3471750893' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:16:38.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:38 smithi145 bash[79878]: audit 2024-03-07T03:16:37.051037+0000 mon.a (mon.0) 194 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:16:38.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:38 smithi145 bash[30906]: cluster 2024-03-07T03:16:37.018560+0000 mon.a (mon.0) 192 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:16:38.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:38 smithi145 bash[30906]: cluster 2024-03-07T03:16:37.044924+0000 mon.a (mon.0) 193 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2024-03-07T03:16:38.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:38 smithi145 bash[30906]: audit 2024-03-07T03:16:37.050279+0000 mon.c (mon.2) 183 : audit 1 from='client.? 172.21.15.145:0/3471750893' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:16:38.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:38 smithi145 bash[30906]: audit 2024-03-07T03:16:37.051037+0000 mon.a (mon.0) 194 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:16:38.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:38 smithi189 bash[23614]: cluster 2024-03-07T03:16:37.018560+0000 mon.a (mon.0) 192 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:16:38.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:38 smithi189 bash[23614]: cluster 2024-03-07T03:16:37.044924+0000 mon.a (mon.0) 193 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2024-03-07T03:16:38.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:38 smithi189 bash[23614]: audit 2024-03-07T03:16:37.050279+0000 mon.c (mon.2) 183 : audit 1 from='client.? 172.21.15.145:0/3471750893' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:16:38.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:38 smithi189 bash[23614]: audit 2024-03-07T03:16:37.051037+0000 mon.a (mon.0) 194 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:16:38.699 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:16:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:38.699 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:16:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:38.699 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:38.700 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:16:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:38.700 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:16:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:38.700 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:16:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:38.700 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:16:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:38.701 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:16:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:38.701 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:38 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:38.701 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:38 smithi145 systemd[1]: Stopping Ceph mon.c for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:16:38.701 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:38 smithi145 bash[30906]: debug 2024-03-07T03:16:38.563+0000 7faaa508e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T03:16:38.701 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:38 smithi145 bash[30906]: debug 2024-03-07T03:16:38.563+0000 7faaa508e700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-03-07T03:16:39.702 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:39 smithi145 bash[82210]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mon-c 2024-03-07T03:16:39.702 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:39 smithi145 bash[82272]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mon-c 2024-03-07T03:16:39.702 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:39 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.c.service: Deactivated successfully. 2024-03-07T03:16:39.703 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:39 smithi145 systemd[1]: Stopped Ceph mon.c for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:16:40.048 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:16:40.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:39 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:40.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:16:39 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:40.059 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:16:39 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:40.059 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:16:39 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:40.060 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:16:39 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:40.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:39 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:40.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:39 smithi145 systemd[1]: Started Ceph mon.c for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:16:40.060 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:16:39 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:40.061 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:16:39 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:40.061 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:16:39 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:41.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.343+0000 7f56dc1d1c80 0 set uid:gid to 167:167 (ceph:ceph) 2024-03-07T03:16:41.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.343+0000 7f56dc1d1c80 0 ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev), process ceph-mon, pid 7 2024-03-07T03:16:41.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.343+0000 7f56dc1d1c80 0 pidfile_write: ignore empty --pid-file 2024-03-07T03:16:41.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 0 load: jerasure load: lrc 2024-03-07T03:16:41.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: RocksDB version: 7.9.2 2024-03-07T03:16:41.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Git sha 0 2024-03-07T03:16:41.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Compile date 2024-03-06 19:59:24 2024-03-07T03:16:41.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: DB SUMMARY 2024-03-07T03:16:41.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: DB Session ID: YMZEJN4DAF30KTTYVW3R 2024-03-07T03:16:41.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: CURRENT file: CURRENT 2024-03-07T03:16:41.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: IDENTITY file: IDENTITY 2024-03-07T03:16:41.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: MANIFEST file: MANIFEST-000010 size: 2630 Bytes 2024-03-07T03:16:41.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000042.sst 2024-03-07T03:16:41.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000040.log size: 2348819 ; 2024-03-07T03:16:41.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.error_if_exists: 0 2024-03-07T03:16:41.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.create_if_missing: 0 2024-03-07T03:16:41.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.paranoid_checks: 1 2024-03-07T03:16:41.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-03-07T03:16:41.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-07T03:16:41.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-07T03:16:41.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.env: 0x560a780c7b20 2024-03-07T03:16:41.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.fs: PosixFileSystem 2024-03-07T03:16:41.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.info_log: 0x560a79397920 2024-03-07T03:16:41.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_file_opening_threads: 16 2024-03-07T03:16:41.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.statistics: (nil) 2024-03-07T03:16:41.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.use_fsync: 0 2024-03-07T03:16:41.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_log_file_size: 0 2024-03-07T03:16:41.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-07T03:16:41.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-03-07T03:16:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.keep_log_file_num: 1000 2024-03-07T03:16:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.recycle_log_file_num: 0 2024-03-07T03:16:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.allow_fallocate: 1 2024-03-07T03:16:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.allow_mmap_reads: 0 2024-03-07T03:16:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.allow_mmap_writes: 0 2024-03-07T03:16:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.use_direct_reads: 0 2024-03-07T03:16:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-07T03:16:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.create_missing_column_families: 0 2024-03-07T03:16:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.db_log_dir: 2024-03-07T03:16:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.wal_dir: 2024-03-07T03:16:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-03-07T03:16:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-03-07T03:16:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-03-07T03:16:41.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-07T03:16:41.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-07T03:16:41.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-03-07T03:16:41.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.advise_random_on_open: 1 2024-03-07T03:16:41.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.db_write_buffer_size: 0 2024-03-07T03:16:41.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.write_buffer_manager: 0x560a7936adc0 2024-03-07T03:16:41.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-07T03:16:41.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-07T03:16:41.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-03-07T03:16:41.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.rate_limiter: (nil) 2024-03-07T03:16:41.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-07T03:16:41.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.wal_recovery_mode: 2 2024-03-07T03:16:41.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.enable_thread_tracking: 0 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.enable_pipelined_write: 0 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.unordered_write: 0 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.row_cache: None 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.wal_filter: None 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.allow_ingest_behind: 0 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.two_write_queues: 0 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.manual_wal_flush: 0 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.wal_compression: 0 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.atomic_flush: 0 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-07T03:16:41.565 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.log_readahead_size: 0 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.best_efforts_recovery: 0 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.allow_data_in_errors: 0 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.db_host_id: __hostname__ 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_background_jobs: 2 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_background_compactions: -1 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_subcompactions: 1 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-07T03:16:41.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_total_wal_size: 0 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_open_files: -1 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.bytes_per_sync: 0 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compaction_readahead_size: 0 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_background_flushes: -1 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Compression algorithms supported: 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: kZSTD supported: 0 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: kXpressCompression supported: 0 2024-03-07T03:16:41.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: kLZ4HCCompression supported: 1 2024-03-07T03:16:41.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: kZlibCompression supported: 1 2024-03-07T03:16:41.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: kSnappyCompression supported: 1 2024-03-07T03:16:41.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: kLZ4Compression supported: 1 2024-03-07T03:16:41.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: kBZip2Compression supported: 0 2024-03-07T03:16:41.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-03-07T03:16:41.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-03-07T03:16:41.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 2024-03-07T03:16:41.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-07T03:16:41.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-07T03:16:41.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.merge_operator: 2024-03-07T03:16:41.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compaction_filter: None 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compaction_filter_factory: None 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.sst_partitioner_factory: None 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560a79397a20) 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: cache_index_and_filter_blocks: 1 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: pin_top_level_index_and_filter: 1 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: index_type: 0 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: data_block_index_type: 0 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: index_shortening: 1 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: data_block_hash_table_util_ratio: 0.750000 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: checksum: 4 2024-03-07T03:16:41.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: no_block_cache: 0 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: block_cache: 0x560a7939b090 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: block_cache_name: BinnedLRUCache 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: block_cache_options: 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: capacity : 536870912 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: num_shard_bits : 4 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: strict_capacity_limit : 0 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: high_pri_pool_ratio: 0.000 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: block_cache_compressed: (nil) 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: persistent_cache: (nil) 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: block_size: 4096 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: block_size_deviation: 10 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: block_restart_interval: 16 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: index_block_restart_interval: 1 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: metadata_block_size: 4096 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: partition_filters: 0 2024-03-07T03:16:41.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: use_delta_encoding: 1 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: filter_policy: bloomfilter 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: whole_key_filtering: 1 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: verify_compression: 0 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: read_amp_bytes_per_bit: 0 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: format_version: 5 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: enable_index_compression: 1 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: block_align: 0 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: max_auto_readahead_size: 262144 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: prepopulate_block_cache: 0 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: initial_auto_readahead_size: 8192 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: num_file_reads_for_auto_readahead: 2 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.write_buffer_size: 33554432 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_write_buffer_number: 2 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compression: NoCompression 2024-03-07T03:16:41.572 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.bottommost_compression: Disabled 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.prefix_extractor: nullptr 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.num_levels: 7 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:16:41.573 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compression_opts.level: 32767 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compression_opts.strategy: 0 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compression_opts.enabled: false 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.target_file_size_base: 67108864 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-07T03:16:41.574 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.arena_block_size: 1048576 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-07T03:16:41.575 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.disable_auto_compactions: 0 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.table_properties_collectors: 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.inplace_update_support: 0 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-03-07T03:16:41.576 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.bloom_locality: 0 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.max_successive_merges: 0 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.paranoid_file_checks: 0 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.force_consistency_checks: 1 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.report_bg_io_stats: 0 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.ttl: 2592000 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.enable_blob_files: false 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.min_blob_size: 0 2024-03-07T03:16:41.577 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.blob_file_size: 268435456 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.blob_file_starting_level: 0 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 44, last_sequence is 24164, log_number is 40,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 40 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 40 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 300caafb-e29e-43ec-ae9d-c3b3f2deb35c 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781401354003, "job": 1, "event": "recovery_started", "wal_files": [40]} 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.347+0000 7f56dc1d1c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #40 mode 2 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.359+0000 7f56dc1d1c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781401364504, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 45, "file_size": 1503919, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 24169, "largest_seqno": 25158, "table_properties": {"data_size": 1498584, "index_size": 3360, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 965, "raw_key_size": 8112, "raw_average_key_size": 22, "raw_value_size": 1489796, "raw_average_value_size": 4070, "num_data_blocks": 146, "num_entries": 366, "num_filter_entries": 366, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1709781401, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "300caafb-e29e-43ec-ae9d-c3b3f2deb35c", "db_session_id": "YMZEJN4DAF30KTTYVW3R", "orig_file_number": 45, "seqno_to_time_mapping": "N/A"}} 2024-03-07T03:16:41.578 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.359+0000 7f56dc1d1c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781401364652, "job": 1, "event": "recovery_finished"} 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.359+0000 7f56dc1d1c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 47 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.359+0000 7f56dc1d1c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.363+0000 7f56dc1d1c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000040.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.363+0000 7f56dc1d1c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x560a79492000 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.363+0000 7f56dc1d1c80 4 rocksdb: DB pointer 0x560a7947c000 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.363+0000 7f56ca93a700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.363+0000 7f56ca93a700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: ** DB Stats ** 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:16:41.579 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:16:41.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: ** Compaction Stats [default] ** 2024-03-07T03:16:41.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:16:41.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-07T03:16:41.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: L0 1/0 1.43 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 339.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-07T03:16:41.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: L6 1/0 11.77 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-07T03:16:41.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Sum 2/0 13.21 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 339.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-07T03:16:41.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 339.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-07T03:16:41.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: ** Compaction Stats [default] ** 2024-03-07T03:16:41.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:16:41.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-07T03:16:41.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 339.8 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-07T03:16:41.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-07T03:16:41.580 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Flush(GB): cumulative 0.001, interval 0.001 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: AddFile(Total Files): cumulative 0, interval 0 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: AddFile(L0 Files): cumulative 0, interval 0 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: AddFile(Keys): cumulative 0, interval 0 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Cumulative compaction: 0.00 GB write, 100.80 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Interval compaction: 0.00 GB write, 100.80 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Block cache BinnedLRUCache@0x560a7939b090#7 capacity: 512.00 MB usage: 5.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: ** File Read Latency Histogram By Level [default] ** 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.363+0000 7f56dc1d1c80 0 starting mon.c rank 2 at public addrs [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] at bind addrs [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.363+0000 7f56dc1d1c80 1 mon.c@-1(???) e3 preinit fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.363+0000 7f56dc1d1c80 0 mon.c@-1(???).mds e10 new map 2024-03-07T03:16:41.581 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.363+0000 7f56dc1d1c80 0 mon.c@-1(???).mds e10 print_map 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: e10 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: enable_multiple, ever_enabled_multiple: 1,1 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: legacy client fscid: 1 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Filesystem 'foo' (1) 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: fs_name foo 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: epoch 10 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: created 2024-03-07T03:08:49.405527+0000 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: modified 2024-03-07T03:09:00.347215+0000 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: tableserver 0 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: root 0 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: session_timeout 60 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: session_autoclose 300 2024-03-07T03:16:41.582 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: max_file_size 1099511627776 2024-03-07T03:16:41.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: max_xattr_size 65536 2024-03-07T03:16:41.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: required_client_features {} 2024-03-07T03:16:41.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: last_failure 0 2024-03-07T03:16:41.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: last_failure_osd_epoch 70 2024-03-07T03:16:41.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T03:16:41.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: max_mds 1 2024-03-07T03:16:41.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: in 0 2024-03-07T03:16:41.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: up {0=24376} 2024-03-07T03:16:41.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: failed 2024-03-07T03:16:41.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: damaged 2024-03-07T03:16:41.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: stopped 2024-03-07T03:16:41.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: data_pools [4] 2024-03-07T03:16:41.583 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: metadata_pool 3 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: inline_data disabled 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: balancer 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: bal_rank_mask -1 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: standby_count_wanted 1 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: [mds.foo.smithi189.zkinbe{0:24376} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] compat {c=[1],r=[1],i=[7ff]}] 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: Standby daemons: 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: [mds.foo.smithi145.zmgvgw{-1:14589} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.145:6834/4165785656,v1:172.21.15.145:6835/4165785656] compat {c=[1],r=[1],i=[7ff]}] 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.367+0000 7f56dc1d1c80 0 mon.c@-1(???).osd e366 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.367+0000 7f56dc1d1c80 0 mon.c@-1(???).osd e366 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.367+0000 7f56dc1d1c80 0 mon.c@-1(???).osd e366 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:16:41.584 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.367+0000 7f56dc1d1c80 0 mon.c@-1(???).osd e366 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:16:41.585 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:41 smithi145 bash[82329]: debug 2024-03-07T03:16:41.367+0000 7f56dc1d1c80 1 mon.c@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2024-03-07T03:16:42.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:16:42 smithi189 bash[59814]: ts=2024-03-07T03:16:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:43.036 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:16:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:16:42] "GET /metrics HTTP/1.1" 200 38695 "" "Prometheus/2.43.0" 2024-03-07T03:16:43.085 INFO:teuthology.orchestra.run.smithi145.stdout:wait for majority of mons to upgrade 2024-03-07T03:16:43.830 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:16:43 smithi189 bash[59814]: ts=2024-03-07T03:16:43.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:43.913 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-03-07T03:16:47.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:46 smithi145 bash[79878]: debug 2024-03-07T03:16:46.639+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-03-07T03:16:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:46 smithi145 bash[79878]: debug 2024-03-07T03:16:46.639+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-03-07T03:16:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:46 smithi145 bash[79878]: debug 2024-03-07T03:16:46.639+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : mon.c (rank 2) addr [v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0] is down (out of quorum) 2024-03-07T03:16:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:46 smithi145 bash[79878]: debug 2024-03-07T03:16:46.639+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:16:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:46 smithi145 bash[79878]: debug 2024-03-07T03:16:46.639+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:46 smithi145 bash[79878]: debug 2024-03-07T03:16:46.639+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:46 smithi145 bash[79878]: debug 2024-03-07T03:16:46.639+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:46 smithi145 bash[79878]: debug 2024-03-07T03:16:46.639+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:46 smithi145 bash[79878]: debug 2024-03-07T03:16:46.639+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:46 smithi145 bash[79878]: debug 2024-03-07T03:16:46.639+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:46 smithi145 bash[79878]: debug 2024-03-07T03:16:46.639+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:46 smithi145 bash[79878]: debug 2024-03-07T03:16:46.639+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:48.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.590263+0000 mon.a (mon.0) 219 : cluster 1 mon.a calling monitor election 2024-03-07T03:16:48.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.631740+0000 mon.a (mon.0) 220 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:16:48.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.639473+0000 mon.a (mon.0) 221 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:16:48.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.640642+0000 mon.a (mon.0) 222 : cluster 0 fsmap foo:1 {0=foo.smithi189.zkinbe=up:active} 1 up:standby 2024-03-07T03:16:48.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.640752+0000 mon.a (mon.0) 223 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2024-03-07T03:16:48.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.641522+0000 mon.a (mon.0) 224 : cluster 0 mgrmap e38: y(active, since 105s), standbys: x 2024-03-07T03:16:48.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.641939+0000 mon.a (mon.0) 225 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-07T03:16:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: debug 2024-03-07T03:16:47.659+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:16:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: debug 2024-03-07T03:16:47.659+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:16:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: debug 2024-03-07T03:16:47.659+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: debug 2024-03-07T03:16:47.659+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: debug 2024-03-07T03:16:47.659+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: debug 2024-03-07T03:16:47.659+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: debug 2024-03-07T03:16:47.659+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:48.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: debug 2024-03-07T03:16:47.659+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:48.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: debug 2024-03-07T03:16:47.659+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:48.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: debug 2024-03-07T03:16:47.659+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:48.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.590263+0000 mon.a (mon.0) 219 : cluster 1 mon.a calling monitor election 2024-03-07T03:16:48.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.631740+0000 mon.a (mon.0) 220 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:16:48.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.639473+0000 mon.a (mon.0) 221 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:16:48.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.640642+0000 mon.a (mon.0) 222 : cluster 0 fsmap foo:1 {0=foo.smithi189.zkinbe=up:active} 1 up:standby 2024-03-07T03:16:48.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.640752+0000 mon.a (mon.0) 223 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2024-03-07T03:16:48.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.641522+0000 mon.a (mon.0) 224 : cluster 0 mgrmap e38: y(active, since 105s), standbys: x 2024-03-07T03:16:48.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:47 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.641939+0000 mon.a (mon.0) 225 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-07T03:16:48.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:47 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.590263+0000 mon.a (mon.0) 219 : cluster 1 mon.a calling monitor election 2024-03-07T03:16:48.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:47 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.631740+0000 mon.a (mon.0) 220 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:16:48.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:47 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.639473+0000 mon.a (mon.0) 221 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:16:48.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:47 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.640642+0000 mon.a (mon.0) 222 : cluster 0 fsmap foo:1 {0=foo.smithi189.zkinbe=up:active} 1 up:standby 2024-03-07T03:16:48.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:47 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.640752+0000 mon.a (mon.0) 223 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2024-03-07T03:16:48.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:47 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.641522+0000 mon.a (mon.0) 224 : cluster 0 mgrmap e38: y(active, since 105s), standbys: x 2024-03-07T03:16:48.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:47 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.641939+0000 mon.a (mon.0) 225 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-07T03:16:48.635 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:16:48.729 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestGroup.group_create (10677 ms) 2024-03-07T03:16:48.729 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestGroup.group_createPP 2024-03-07T03:16:48.756 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestGroup.group_createPP (28 ms) 2024-03-07T03:16:48.756 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestGroup.add_image 2024-03-07T03:16:48.823 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestGroup.add_image (67 ms) 2024-03-07T03:16:48.823 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestGroup.add_imagePP 2024-03-07T03:16:48.898 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestGroup.add_imagePP (75 ms) 2024-03-07T03:16:48.898 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestGroup.add_snapshot 2024-03-07T03:16:49.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: cluster 2024-03-07T03:16:41.584225+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-03-07T03:16:49.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.588581+0000 mon.b (mon.1) 94 : cluster 1 mon.b calling monitor election 2024-03-07T03:16:49.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.665362+0000 mon.a (mon.0) 226 : cluster 4 Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:16:49.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.665410+0000 mon.a (mon.0) 227 : cluster 4 [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:16:49.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.665432+0000 mon.a (mon.0) 228 : cluster 4 osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.665507+0000 mon.a (mon.0) 229 : cluster 4 osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.665782+0000 mon.a (mon.0) 230 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.665807+0000 mon.a (mon.0) 231 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.665835+0000 mon.a (mon.0) 232 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.665871+0000 mon.a (mon.0) 233 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.665889+0000 mon.a (mon.0) 234 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.665913+0000 mon.a (mon.0) 235 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: audit 2024-03-07T03:16:47.678043+0000 mon.a (mon.0) 236 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:49.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: cluster 2024-03-07T03:16:47.680978+0000 mon.a (mon.0) 237 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2024-03-07T03:16:49.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: audit 2024-03-07T03:16:47.682566+0000 mon.a (mon.0) 238 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:49.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[79878]: audit 2024-03-07T03:16:47.690720+0000 mon.a (mon.0) 239 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:49.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: cluster 2024-03-07T03:16:41.584225+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-03-07T03:16:49.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.588581+0000 mon.b (mon.1) 94 : cluster 1 mon.b calling monitor election 2024-03-07T03:16:49.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.665362+0000 mon.a (mon.0) 226 : cluster 4 Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:16:49.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.665410+0000 mon.a (mon.0) 227 : cluster 4 [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:16:49.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.665432+0000 mon.a (mon.0) 228 : cluster 4 osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.665507+0000 mon.a (mon.0) 229 : cluster 4 osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.665782+0000 mon.a (mon.0) 230 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.665807+0000 mon.a (mon.0) 231 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.665835+0000 mon.a (mon.0) 232 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.665871+0000 mon.a (mon.0) 233 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.665889+0000 mon.a (mon.0) 234 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.665913+0000 mon.a (mon.0) 235 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: audit 2024-03-07T03:16:47.678043+0000 mon.a (mon.0) 236 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:49.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: cluster 2024-03-07T03:16:47.680978+0000 mon.a (mon.0) 237 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2024-03-07T03:16:49.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: audit 2024-03-07T03:16:47.682566+0000 mon.a (mon.0) 238 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:49.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:48 smithi145 bash[82329]: audit 2024-03-07T03:16:47.690720+0000 mon.a (mon.0) 239 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:49.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: cluster 2024-03-07T03:16:41.584225+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-03-07T03:16:49.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.588581+0000 mon.b (mon.1) 94 : cluster 1 mon.b calling monitor election 2024-03-07T03:16:49.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.665362+0000 mon.a (mon.0) 226 : cluster 4 Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:16:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.665410+0000 mon.a (mon.0) 227 : cluster 4 [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:16:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.665432+0000 mon.a (mon.0) 228 : cluster 4 osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.665507+0000 mon.a (mon.0) 229 : cluster 4 osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.665782+0000 mon.a (mon.0) 230 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.665807+0000 mon.a (mon.0) 231 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.665835+0000 mon.a (mon.0) 232 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.665871+0000 mon.a (mon.0) 233 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.665889+0000 mon.a (mon.0) 234 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.665913+0000 mon.a (mon.0) 235 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:16:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: audit 2024-03-07T03:16:47.678043+0000 mon.a (mon.0) 236 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: cluster 2024-03-07T03:16:47.680978+0000 mon.a (mon.0) 237 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2024-03-07T03:16:49.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: audit 2024-03-07T03:16:47.682566+0000 mon.a (mon.0) 238 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:49.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:48 smithi189 bash[23614]: audit 2024-03-07T03:16:47.690720+0000 mon.a (mon.0) 239 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:49.951 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:49 smithi145 bash[79878]: cluster 2024-03-07T03:16:48.561887+0000 mgr.y (mgr.34107) 74 : cluster 0 pgmap v141: 105 pgs: 105 active+clean; 582 KiB data, 365 MiB used, 715 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-03-07T03:16:49.951 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:49 smithi145 bash[79878]: cluster 2024-03-07T03:16:48.694991+0000 mon.a (mon.0) 240 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2024-03-07T03:16:49.951 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:49 smithi145 bash[82329]: cluster 2024-03-07T03:16:48.561887+0000 mgr.y (mgr.34107) 74 : cluster 0 pgmap v141: 105 pgs: 105 active+clean; 582 KiB data, 365 MiB used, 715 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-03-07T03:16:49.951 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:49 smithi145 bash[82329]: cluster 2024-03-07T03:16:48.694991+0000 mon.a (mon.0) 240 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2024-03-07T03:16:50.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:49 smithi189 bash[23614]: cluster 2024-03-07T03:16:48.561887+0000 mgr.y (mgr.34107) 74 : cluster 0 pgmap v141: 105 pgs: 105 active+clean; 582 KiB data, 365 MiB used, 715 GiB / 715 GiB avail; 102 B/s wr, 0 op/s 2024-03-07T03:16:50.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:49 smithi189 bash[23614]: cluster 2024-03-07T03:16:48.694991+0000 mon.a (mon.0) 240 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2024-03-07T03:16:51.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:50 smithi145 bash[79878]: cluster 2024-03-07T03:16:49.708603+0000 mon.a (mon.0) 241 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2024-03-07T03:16:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:50 smithi145 bash[82329]: cluster 2024-03-07T03:16:49.708603+0000 mon.a (mon.0) 241 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2024-03-07T03:16:51.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:50 smithi189 bash[23614]: cluster 2024-03-07T03:16:49.708603+0000 mon.a (mon.0) 241 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2024-03-07T03:16:51.713 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:16:51 smithi145 bash[72238]: debug 2024-03-07T03:16:51.375+0000 7f318c55a700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-03-07T03:16:52.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:51 smithi145 bash[79878]: cluster 2024-03-07T03:16:50.562590+0000 mgr.y (mgr.34107) 75 : cluster 0 pgmap v144: 105 pgs: 105 active+clean; 582 KiB data, 365 MiB used, 715 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-03-07T03:16:52.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:51 smithi145 bash[79878]: cluster 2024-03-07T03:16:50.725812+0000 mon.a (mon.0) 242 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2024-03-07T03:16:52.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:51 smithi145 bash[82329]: cluster 2024-03-07T03:16:50.562590+0000 mgr.y (mgr.34107) 75 : cluster 0 pgmap v144: 105 pgs: 105 active+clean; 582 KiB data, 365 MiB used, 715 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-03-07T03:16:52.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:51 smithi145 bash[82329]: cluster 2024-03-07T03:16:50.725812+0000 mon.a (mon.0) 242 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2024-03-07T03:16:52.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:51 smithi189 bash[23614]: cluster 2024-03-07T03:16:50.562590+0000 mgr.y (mgr.34107) 75 : cluster 0 pgmap v144: 105 pgs: 105 active+clean; 582 KiB data, 365 MiB used, 715 GiB / 715 GiB avail; 170 B/s wr, 0 op/s 2024-03-07T03:16:52.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:51 smithi189 bash[23614]: cluster 2024-03-07T03:16:50.725812+0000 mon.a (mon.0) 242 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2024-03-07T03:16:52.302 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 2 2024-03-07T03:16:52.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:16:52 smithi189 bash[59814]: ts=2024-03-07T03:16:52.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:53.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:16:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:16:52] "GET /metrics HTTP/1.1" 200 36248 "" "Prometheus/2.43.0" 2024-03-07T03:16:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:52 smithi145 bash[79878]: cluster 2024-03-07T03:16:51.719755+0000 mon.a (mon.0) 243 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2024-03-07T03:16:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:52 smithi145 bash[79878]: audit 2024-03-07T03:16:51.969388+0000 mon.a (mon.0) 244 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:52 smithi145 bash[79878]: audit 2024-03-07T03:16:51.975215+0000 mon.a (mon.0) 245 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:52 smithi145 bash[79878]: audit 2024-03-07T03:16:52.296102+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.145:0/1737640254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:53.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:52 smithi145 bash[82329]: cluster 2024-03-07T03:16:51.719755+0000 mon.a (mon.0) 243 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2024-03-07T03:16:53.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:52 smithi145 bash[82329]: audit 2024-03-07T03:16:51.969388+0000 mon.a (mon.0) 244 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:53.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:52 smithi145 bash[82329]: audit 2024-03-07T03:16:51.975215+0000 mon.a (mon.0) 245 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:53.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:52 smithi145 bash[82329]: audit 2024-03-07T03:16:52.296102+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.145:0/1737640254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:53.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:52 smithi189 bash[23614]: cluster 2024-03-07T03:16:51.719755+0000 mon.a (mon.0) 243 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2024-03-07T03:16:53.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:52 smithi189 bash[23614]: audit 2024-03-07T03:16:51.969388+0000 mon.a (mon.0) 244 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:53.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:52 smithi189 bash[23614]: audit 2024-03-07T03:16:51.975215+0000 mon.a (mon.0) 245 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:53.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:52 smithi189 bash[23614]: audit 2024-03-07T03:16:52.296102+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.145:0/1737640254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:53.130 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch ps' 2024-03-07T03:16:53.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:53 smithi189 bash[23614]: cluster 2024-03-07T03:16:52.563050+0000 mgr.y (mgr.34107) 76 : cluster 0 pgmap v147: 105 pgs: 105 active+clean; 582 KiB data, 365 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:16:53.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:53 smithi189 bash[23614]: cluster 2024-03-07T03:16:52.742858+0000 mon.a (mon.0) 246 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2024-03-07T03:16:53.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:53 smithi189 bash[23614]: audit 2024-03-07T03:16:52.914236+0000 mon.a (mon.0) 247 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:53.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:53 smithi189 bash[23614]: audit 2024-03-07T03:16:52.921574+0000 mon.a (mon.0) 248 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:53.830 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:16:53 smithi189 bash[59814]: ts=2024-03-07T03:16:53.513Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:54.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:53 smithi145 bash[79878]: cluster 2024-03-07T03:16:52.563050+0000 mgr.y (mgr.34107) 76 : cluster 0 pgmap v147: 105 pgs: 105 active+clean; 582 KiB data, 365 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:16:54.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:53 smithi145 bash[79878]: cluster 2024-03-07T03:16:52.742858+0000 mon.a (mon.0) 246 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2024-03-07T03:16:54.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:53 smithi145 bash[79878]: audit 2024-03-07T03:16:52.914236+0000 mon.a (mon.0) 247 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:54.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:53 smithi145 bash[79878]: audit 2024-03-07T03:16:52.921574+0000 mon.a (mon.0) 248 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:54.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:53 smithi145 bash[82329]: cluster 2024-03-07T03:16:52.563050+0000 mgr.y (mgr.34107) 76 : cluster 0 pgmap v147: 105 pgs: 105 active+clean; 582 KiB data, 365 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:16:54.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:53 smithi145 bash[82329]: cluster 2024-03-07T03:16:52.742858+0000 mon.a (mon.0) 246 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2024-03-07T03:16:54.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:53 smithi145 bash[82329]: audit 2024-03-07T03:16:52.914236+0000 mon.a (mon.0) 247 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:54.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:53 smithi145 bash[82329]: audit 2024-03-07T03:16:52.921574+0000 mon.a (mon.0) 248 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:16:54.805 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestGroup.add_snapshot (5907 ms) 2024-03-07T03:16:54.805 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestGroup.add_snapshotPP 2024-03-07T03:16:55.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:54 smithi145 bash[79878]: cluster 2024-03-07T03:16:53.742628+0000 mon.a (mon.0) 249 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2024-03-07T03:16:55.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:54 smithi145 bash[82329]: cluster 2024-03-07T03:16:53.742628+0000 mon.a (mon.0) 249 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2024-03-07T03:16:55.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:54 smithi189 bash[23614]: cluster 2024-03-07T03:16:53.742628+0000 mon.a (mon.0) 249 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2024-03-07T03:16:56.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:55 smithi145 bash[79878]: cluster 2024-03-07T03:16:54.563730+0000 mgr.y (mgr.34107) 77 : cluster 0 pgmap v150: 105 pgs: 105 active+clean; 582 KiB data, 374 MiB used, 715 GiB / 715 GiB avail; 163 KiB/s rd, 18 KiB/s wr, 220 op/s 2024-03-07T03:16:56.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:55 smithi145 bash[79878]: cluster 2024-03-07T03:16:54.760988+0000 mon.a (mon.0) 250 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2024-03-07T03:16:56.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:55 smithi145 bash[82329]: cluster 2024-03-07T03:16:54.563730+0000 mgr.y (mgr.34107) 77 : cluster 0 pgmap v150: 105 pgs: 105 active+clean; 582 KiB data, 374 MiB used, 715 GiB / 715 GiB avail; 163 KiB/s rd, 18 KiB/s wr, 220 op/s 2024-03-07T03:16:56.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:55 smithi145 bash[82329]: cluster 2024-03-07T03:16:54.760988+0000 mon.a (mon.0) 250 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2024-03-07T03:16:56.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:55 smithi189 bash[23614]: cluster 2024-03-07T03:16:54.563730+0000 mgr.y (mgr.34107) 77 : cluster 0 pgmap v150: 105 pgs: 105 active+clean; 582 KiB data, 374 MiB used, 715 GiB / 715 GiB avail; 163 KiB/s rd, 18 KiB/s wr, 220 op/s 2024-03-07T03:16:56.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:55 smithi189 bash[23614]: cluster 2024-03-07T03:16:54.760988+0000 mon.a (mon.0) 250 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2024-03-07T03:16:57.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:56 smithi145 bash[79878]: cluster 2024-03-07T03:16:55.781264+0000 mon.a (mon.0) 251 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2024-03-07T03:16:57.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:56 smithi145 bash[82329]: cluster 2024-03-07T03:16:55.781264+0000 mon.a (mon.0) 251 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2024-03-07T03:16:57.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:56 smithi189 bash[23614]: cluster 2024-03-07T03:16:55.781264+0000 mon.a (mon.0) 251 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2024-03-07T03:16:57.866 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:16:58.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:57 smithi145 bash[79878]: cluster 2024-03-07T03:16:56.564422+0000 mgr.y (mgr.34107) 78 : cluster 0 pgmap v153: 105 pgs: 105 active+clean; 582 KiB data, 374 MiB used, 715 GiB / 715 GiB avail; 163 KiB/s rd, 18 KiB/s wr, 220 op/s 2024-03-07T03:16:58.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:57 smithi145 bash[79878]: cluster 2024-03-07T03:16:56.796935+0000 mon.a (mon.0) 252 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2024-03-07T03:16:58.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:57 smithi145 bash[79878]: cluster 2024-03-07T03:16:57.269303+0000 mon.a (mon.0) 253 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2024-03-07T03:16:58.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:57 smithi145 bash[82329]: cluster 2024-03-07T03:16:56.564422+0000 mgr.y (mgr.34107) 78 : cluster 0 pgmap v153: 105 pgs: 105 active+clean; 582 KiB data, 374 MiB used, 715 GiB / 715 GiB avail; 163 KiB/s rd, 18 KiB/s wr, 220 op/s 2024-03-07T03:16:58.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:57 smithi145 bash[82329]: cluster 2024-03-07T03:16:56.796935+0000 mon.a (mon.0) 252 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2024-03-07T03:16:58.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:57 smithi145 bash[82329]: cluster 2024-03-07T03:16:57.269303+0000 mon.a (mon.0) 253 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2024-03-07T03:16:58.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:57 smithi189 bash[23614]: cluster 2024-03-07T03:16:56.564422+0000 mgr.y (mgr.34107) 78 : cluster 0 pgmap v153: 105 pgs: 105 active+clean; 582 KiB data, 374 MiB used, 715 GiB / 715 GiB avail; 163 KiB/s rd, 18 KiB/s wr, 220 op/s 2024-03-07T03:16:58.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:57 smithi189 bash[23614]: cluster 2024-03-07T03:16:56.796935+0000 mon.a (mon.0) 252 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2024-03-07T03:16:58.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:57 smithi189 bash[23614]: cluster 2024-03-07T03:16:57.269303+0000 mon.a (mon.0) 253 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2024-03-07T03:16:59.553 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:59 smithi145 bash[79878]: cluster 2024-03-07T03:16:58.273279+0000 mon.a (mon.0) 254 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2024-03-07T03:16:59.553 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:16:59 smithi145 bash[79878]: cluster 2024-03-07T03:16:58.565702+0000 mgr.y (mgr.34107) 79 : cluster 0 pgmap v157: 105 pgs: 105 active+clean; 583 KiB data, 375 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 13 KiB/s wr, 169 op/s 2024-03-07T03:16:59.554 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:59 smithi145 bash[82329]: cluster 2024-03-07T03:16:58.273279+0000 mon.a (mon.0) 254 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2024-03-07T03:16:59.554 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:16:59 smithi145 bash[82329]: cluster 2024-03-07T03:16:58.565702+0000 mgr.y (mgr.34107) 79 : cluster 0 pgmap v157: 105 pgs: 105 active+clean; 583 KiB data, 375 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 13 KiB/s wr, 169 op/s 2024-03-07T03:16:59.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:59 smithi189 bash[23614]: cluster 2024-03-07T03:16:58.273279+0000 mon.a (mon.0) 254 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2024-03-07T03:16:59.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:16:59 smithi189 bash[23614]: cluster 2024-03-07T03:16:58.565702+0000 mgr.y (mgr.34107) 79 : cluster 0 pgmap v157: 105 pgs: 105 active+clean; 583 KiB data, 375 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 13 KiB/s wr, 169 op/s 2024-03-07T03:17:00.363 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestGroup.add_snapshotPP (5558 ms) 2024-03-07T03:17:00.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:00 smithi145 bash[79878]: cluster 2024-03-07T03:16:59.287687+0000 mon.a (mon.0) 255 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2024-03-07T03:17:00.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:00 smithi145 bash[82329]: cluster 2024-03-07T03:16:59.287687+0000 mon.a (mon.0) 255 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2024-03-07T03:17:00.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:00 smithi189 bash[23614]: cluster 2024-03-07T03:16:59.287687+0000 mon.a (mon.0) 255 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2024-03-07T03:17:01.308 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 6 tests from TestGroup (22312 ms total) 2024-03-07T03:17:01.308 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:17:01.308 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 31 tests from TestDeepCopy 2024-03-07T03:17:01.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:01 smithi145 bash[79878]: cluster 2024-03-07T03:17:00.287020+0000 mon.a (mon.0) 256 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2024-03-07T03:17:01.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:01 smithi145 bash[79878]: cluster 2024-03-07T03:17:00.566453+0000 mgr.y (mgr.34107) 80 : cluster 0 pgmap v160: 105 pgs: 105 active+clean; 583 KiB data, 375 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 13 KiB/s wr, 170 op/s 2024-03-07T03:17:01.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:01 smithi145 bash[82329]: cluster 2024-03-07T03:17:00.287020+0000 mon.a (mon.0) 256 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2024-03-07T03:17:01.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:01 smithi145 bash[82329]: cluster 2024-03-07T03:17:00.566453+0000 mgr.y (mgr.34107) 80 : cluster 0 pgmap v160: 105 pgs: 105 active+clean; 583 KiB data, 375 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 13 KiB/s wr, 170 op/s 2024-03-07T03:17:01.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:01 smithi189 bash[23614]: cluster 2024-03-07T03:17:00.287020+0000 mon.a (mon.0) 256 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2024-03-07T03:17:01.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:01 smithi189 bash[23614]: cluster 2024-03-07T03:17:00.566453+0000 mgr.y (mgr.34107) 80 : cluster 0 pgmap v160: 105 pgs: 105 active+clean; 583 KiB data, 375 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 13 KiB/s wr, 170 op/s 2024-03-07T03:17:02.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:02 smithi145 bash[79878]: cluster 2024-03-07T03:17:01.313369+0000 mon.a (mon.0) 257 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2024-03-07T03:17:02.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:02 smithi145 bash[82329]: cluster 2024-03-07T03:17:01.313369+0000 mon.a (mon.0) 257 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2024-03-07T03:17:02.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:02 smithi189 bash[23614]: cluster 2024-03-07T03:17:01.313369+0000 mon.a (mon.0) 257 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2024-03-07T03:17:02.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:02 smithi189 bash[59814]: ts=2024-03-07T03:17:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:03.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:17:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:17:02] "GET /metrics HTTP/1.1" 200 36244 "" "Prometheus/2.43.0" 2024-03-07T03:17:03.194 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-07T03:17:03.194 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (9m) 11s ago 10m 14.6M - 0.25.0 c8568f914cd2 cfa1e64a6007 2024-03-07T03:17:03.194 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi189 *:3000 running (10m) 36s ago 10m 79.4M - 9.4.7 954c08fa6188 89badf11bd11 2024-03-07T03:17:03.194 INFO:teuthology.orchestra.run.smithi145.stdout:mds.foo.smithi145.zmgvgw smithi145 running (8m) 11s ago 8m 14.5M - 18.2.1-718-gae75fc9d 0164bb8a7c67 4ded79fef80e 2024-03-07T03:17:03.194 INFO:teuthology.orchestra.run.smithi145.stdout:mds.foo.smithi189.zkinbe smithi189 running (8m) 36s ago 8m 17.0M - 18.2.1-718-gae75fc9d 0164bb8a7c67 6589deb5c181 2024-03-07T03:17:03.194 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi189 *:8443,9283,8765 running (2m) 36s ago 18m 436M - 19.0.0-1905-gea9a5688 fdb83767c1ab acef232ce9b9 2024-03-07T03:17:03.194 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (3m) 11s ago 21m 487M - 19.0.0-1905-gea9a5688 fdb83767c1ab 2cae6fe389e0 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (2m) 11s ago 21m 43.0M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 8a2f4984bf09 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi189 running (19m) 36s ago 19m 54.4M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 5746e42ef7c9 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (23s) 11s ago 19m 18.3M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 1c9cf0604d03 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (10m) 11s ago 10m 9088k - 1.5.0 0da6a335fe13 895e660f39b6 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi189 *:9100 running (10m) 36s ago 10m 8607k - 1.5.0 0da6a335fe13 ab7ffb60c1e8 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (17m) 11s ago 17m 127M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 4485c22f1548 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (17m) 11s ago 17m 112M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 f21d0bf7ebc8 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (16m) 11s ago 16m 118M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 9e8c1bf873ba 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (15m) 11s ago 15m 115M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 c209b1a43303 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi189 running (14m) 36s ago 14m 136M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 8845b4f5216a 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi189 running (13m) 36s ago 13m 129M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 44e5411eb07c 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi189 running (12m) 36s ago 12m 171M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 c2aeea0466c7 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi189 running (12m) 36s ago 12m 121M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 bf101bfb6800 2024-03-07T03:17:03.195 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi189 *:9095 running (2m) 36s ago 11m 49.7M - 2.43.0 a07b618ecd1d 144e5cadeea3 2024-03-07T03:17:03.318 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Empty 2024-03-07T03:17:03.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:03 smithi145 bash[79878]: cluster 2024-03-07T03:17:02.322180+0000 mon.a (mon.0) 258 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2024-03-07T03:17:03.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:03 smithi145 bash[79878]: audit 2024-03-07T03:17:02.326202+0000 mon.a (mon.0) 259 : audit 1 from='client.? 172.21.15.145:0/2612018213' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:17:03.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:03 smithi145 bash[79878]: cluster 2024-03-07T03:17:02.567134+0000 mgr.y (mgr.34107) 81 : cluster 0 pgmap v163: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 375 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:03.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:03 smithi145 bash[79878]: audit 2024-03-07T03:17:02.637390+0000 mon.a (mon.0) 260 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:03.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:03 smithi145 bash[79878]: audit 2024-03-07T03:17:02.639816+0000 mon.a (mon.0) 261 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:03 smithi145 bash[82329]: cluster 2024-03-07T03:17:02.322180+0000 mon.a (mon.0) 258 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2024-03-07T03:17:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:03 smithi145 bash[82329]: audit 2024-03-07T03:17:02.326202+0000 mon.a (mon.0) 259 : audit 1 from='client.? 172.21.15.145:0/2612018213' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:17:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:03 smithi145 bash[82329]: cluster 2024-03-07T03:17:02.567134+0000 mgr.y (mgr.34107) 81 : cluster 0 pgmap v163: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 375 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:03 smithi145 bash[82329]: audit 2024-03-07T03:17:02.637390+0000 mon.a (mon.0) 260 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:03.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:03 smithi145 bash[82329]: audit 2024-03-07T03:17:02.639816+0000 mon.a (mon.0) 261 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:03.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:03 smithi189 bash[59814]: ts=2024-03-07T03:17:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:03.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:03 smithi189 bash[23614]: cluster 2024-03-07T03:17:02.322180+0000 mon.a (mon.0) 258 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2024-03-07T03:17:03.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:03 smithi189 bash[23614]: audit 2024-03-07T03:17:02.326202+0000 mon.a (mon.0) 259 : audit 1 from='client.? 172.21.15.145:0/2612018213' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:17:03.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:03 smithi189 bash[23614]: cluster 2024-03-07T03:17:02.567134+0000 mgr.y (mgr.34107) 81 : cluster 0 pgmap v163: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 375 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:03.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:03 smithi189 bash[23614]: audit 2024-03-07T03:17:02.637390+0000 mon.a (mon.0) 260 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:03.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:03 smithi189 bash[23614]: audit 2024-03-07T03:17:02.639816+0000 mon.a (mon.0) 261 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:04.174 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade pause' 2024-03-07T03:17:04.532 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:04 smithi145 bash[79878]: audit 2024-03-07T03:17:03.188377+0000 mgr.y (mgr.34107) 82 : audit 0 from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:17:04.532 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:04 smithi145 bash[79878]: cluster 2024-03-07T03:17:03.312742+0000 mon.a (mon.0) 262 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:17:04.533 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:04 smithi145 bash[79878]: audit 2024-03-07T03:17:03.320548+0000 mon.a (mon.0) 263 : audit 1 from='client.? 172.21.15.145:0/2612018213' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:17:04.533 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:04 smithi145 bash[79878]: cluster 2024-03-07T03:17:03.329666+0000 mon.a (mon.0) 264 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2024-03-07T03:17:04.533 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:04 smithi145 bash[82329]: audit 2024-03-07T03:17:03.188377+0000 mgr.y (mgr.34107) 82 : audit 0 from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:17:04.533 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:04 smithi145 bash[82329]: cluster 2024-03-07T03:17:03.312742+0000 mon.a (mon.0) 262 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:17:04.533 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:04 smithi145 bash[82329]: audit 2024-03-07T03:17:03.320548+0000 mon.a (mon.0) 263 : audit 1 from='client.? 172.21.15.145:0/2612018213' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:17:04.533 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:04 smithi145 bash[82329]: cluster 2024-03-07T03:17:03.329666+0000 mon.a (mon.0) 264 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2024-03-07T03:17:04.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:04 smithi189 bash[23614]: audit 2024-03-07T03:17:03.188377+0000 mgr.y (mgr.34107) 82 : audit 0 from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:17:04.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:04 smithi189 bash[23614]: cluster 2024-03-07T03:17:03.312742+0000 mon.a (mon.0) 262 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:17:04.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:04 smithi189 bash[23614]: audit 2024-03-07T03:17:03.320548+0000 mon.a (mon.0) 263 : audit 1 from='client.? 172.21.15.145:0/2612018213' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:17:04.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:04 smithi189 bash[23614]: cluster 2024-03-07T03:17:03.329666+0000 mon.a (mon.0) 264 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2024-03-07T03:17:05.492 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.Empty (2175 ms) 2024-03-07T03:17:05.492 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.NoSnaps 2024-03-07T03:17:05.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[79878]: cluster 2024-03-07T03:17:04.325757+0000 mon.a (mon.0) 265 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2024-03-07T03:17:05.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[79878]: cluster 2024-03-07T03:17:04.567823+0000 mgr.y (mgr.34107) 83 : cluster 0 pgmap v166: 105 pgs: 105 active+clean; 582 KiB data, 375 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:05.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[79878]: audit 2024-03-07T03:17:05.198231+0000 mon.a (mon.0) 266 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[79878]: audit 2024-03-07T03:17:05.211793+0000 mon.a (mon.0) 267 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[79878]: audit 2024-03-07T03:17:05.214397+0000 mon.a (mon.0) 268 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[79878]: audit 2024-03-07T03:17:05.216339+0000 mon.a (mon.0) 269 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:17:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[79878]: audit 2024-03-07T03:17:05.228657+0000 mon.a (mon.0) 270 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[79878]: audit 2024-03-07T03:17:05.325504+0000 mon.a (mon.0) 271 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:05.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[82329]: cluster 2024-03-07T03:17:04.325757+0000 mon.a (mon.0) 265 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2024-03-07T03:17:05.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[82329]: cluster 2024-03-07T03:17:04.567823+0000 mgr.y (mgr.34107) 83 : cluster 0 pgmap v166: 105 pgs: 105 active+clean; 582 KiB data, 375 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:05.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[82329]: audit 2024-03-07T03:17:05.198231+0000 mon.a (mon.0) 266 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:05.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[82329]: audit 2024-03-07T03:17:05.211793+0000 mon.a (mon.0) 267 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:05.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[82329]: audit 2024-03-07T03:17:05.214397+0000 mon.a (mon.0) 268 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:05.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[82329]: audit 2024-03-07T03:17:05.216339+0000 mon.a (mon.0) 269 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:17:05.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[82329]: audit 2024-03-07T03:17:05.228657+0000 mon.a (mon.0) 270 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:05.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:05 smithi145 bash[82329]: audit 2024-03-07T03:17:05.325504+0000 mon.a (mon.0) 271 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:05.818 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:05 smithi189 bash[23614]: cluster 2024-03-07T03:17:04.325757+0000 mon.a (mon.0) 265 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2024-03-07T03:17:05.819 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:05 smithi189 bash[23614]: cluster 2024-03-07T03:17:04.567823+0000 mgr.y (mgr.34107) 83 : cluster 0 pgmap v166: 105 pgs: 105 active+clean; 582 KiB data, 375 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:05.819 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:05 smithi189 bash[23614]: audit 2024-03-07T03:17:05.198231+0000 mon.a (mon.0) 266 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:05.819 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:05 smithi189 bash[23614]: audit 2024-03-07T03:17:05.211793+0000 mon.a (mon.0) 267 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:05.819 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:05 smithi189 bash[23614]: audit 2024-03-07T03:17:05.214397+0000 mon.a (mon.0) 268 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:05.819 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:05 smithi189 bash[23614]: audit 2024-03-07T03:17:05.216339+0000 mon.a (mon.0) 269 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:17:05.819 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:05 smithi189 bash[23614]: audit 2024-03-07T03:17:05.228657+0000 mon.a (mon.0) 270 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:05.819 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:05 smithi189 bash[23614]: audit 2024-03-07T03:17:05.325504+0000 mon.a (mon.0) 271 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:05.849 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.NoSnaps (357 ms) 2024-03-07T03:17:05.849 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Snaps 2024-03-07T03:17:06.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:06 smithi189 bash[23614]: cluster 2024-03-07T03:17:05.330531+0000 mon.a (mon.0) 272 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:17:06.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:06 smithi189 bash[23614]: audit 2024-03-07T03:17:05.330977+0000 mon.a (mon.0) 273 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:17:06.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:06 smithi189 bash[23614]: cluster 2024-03-07T03:17:05.342391+0000 mon.a (mon.0) 274 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2024-03-07T03:17:06.780 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:06 smithi189 bash[23614]: audit 2024-03-07T03:17:05.347308+0000 mon.a (mon.0) 275 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:17:06.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:06 smithi189 bash[23614]: audit 2024-03-07T03:17:05.360529+0000 mon.a (mon.0) 276 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-07T03:17:06.781 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:06 smithi189 bash[23614]: cephadm 2024-03-07T03:17:05.362734+0000 mgr.y (mgr.34107) 84 : cephadm 1 Upgrade: It appears safe to stop mon.b 2024-03-07T03:17:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:06 smithi145 bash[79878]: cluster 2024-03-07T03:17:05.330531+0000 mon.a (mon.0) 272 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:17:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:06 smithi145 bash[79878]: audit 2024-03-07T03:17:05.330977+0000 mon.a (mon.0) 273 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:17:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:06 smithi145 bash[79878]: cluster 2024-03-07T03:17:05.342391+0000 mon.a (mon.0) 274 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2024-03-07T03:17:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:06 smithi145 bash[79878]: audit 2024-03-07T03:17:05.347308+0000 mon.a (mon.0) 275 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:17:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:06 smithi145 bash[79878]: audit 2024-03-07T03:17:05.360529+0000 mon.a (mon.0) 276 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-07T03:17:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:06 smithi145 bash[79878]: cephadm 2024-03-07T03:17:05.362734+0000 mgr.y (mgr.34107) 84 : cephadm 1 Upgrade: It appears safe to stop mon.b 2024-03-07T03:17:06.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:06 smithi145 bash[82329]: cluster 2024-03-07T03:17:05.330531+0000 mon.a (mon.0) 272 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:17:06.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:06 smithi145 bash[82329]: audit 2024-03-07T03:17:05.330977+0000 mon.a (mon.0) 273 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:17:06.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:06 smithi145 bash[82329]: cluster 2024-03-07T03:17:05.342391+0000 mon.a (mon.0) 274 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2024-03-07T03:17:06.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:06 smithi145 bash[82329]: audit 2024-03-07T03:17:05.347308+0000 mon.a (mon.0) 275 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:17:06.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:06 smithi145 bash[82329]: audit 2024-03-07T03:17:05.360529+0000 mon.a (mon.0) 276 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-07T03:17:06.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:06 smithi145 bash[82329]: cephadm 2024-03-07T03:17:05.362734+0000 mgr.y (mgr.34107) 84 : cephadm 1 Upgrade: It appears safe to stop mon.b 2024-03-07T03:17:07.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:07 smithi145 bash[79878]: cluster 2024-03-07T03:17:06.358472+0000 mon.a (mon.0) 277 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2024-03-07T03:17:07.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:07 smithi145 bash[79878]: cluster 2024-03-07T03:17:06.568490+0000 mgr.y (mgr.34107) 85 : cluster 0 pgmap v169: 105 pgs: 105 active+clean; 582 KiB data, 375 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:07.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:07 smithi145 bash[82329]: cluster 2024-03-07T03:17:06.358472+0000 mon.a (mon.0) 277 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2024-03-07T03:17:07.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:07 smithi145 bash[82329]: cluster 2024-03-07T03:17:06.568490+0000 mgr.y (mgr.34107) 85 : cluster 0 pgmap v169: 105 pgs: 105 active+clean; 582 KiB data, 375 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:07.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:07 smithi189 bash[23614]: cluster 2024-03-07T03:17:06.358472+0000 mon.a (mon.0) 277 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2024-03-07T03:17:07.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:07 smithi189 bash[23614]: cluster 2024-03-07T03:17:06.568490+0000 mgr.y (mgr.34107) 85 : cluster 0 pgmap v169: 105 pgs: 105 active+clean; 582 KiB data, 375 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:08.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:08 smithi189 bash[23614]: cluster 2024-03-07T03:17:07.381976+0000 mon.a (mon.0) 278 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2024-03-07T03:17:08.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:08 smithi189 bash[23614]: cephadm 2024-03-07T03:17:07.733829+0000 mgr.y (mgr.34107) 86 : cephadm 1 Upgrade: Updating mon.b 2024-03-07T03:17:08.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:08 smithi189 bash[23614]: audit 2024-03-07T03:17:07.741237+0000 mon.a (mon.0) 279 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:08.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:08 smithi189 bash[23614]: audit 2024-03-07T03:17:07.745459+0000 mon.a (mon.0) 280 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:17:08.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:08 smithi189 bash[23614]: audit 2024-03-07T03:17:07.746917+0000 mon.a (mon.0) 281 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:17:08.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:08 smithi189 bash[23614]: audit 2024-03-07T03:17:07.748328+0000 mon.a (mon.0) 282 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:08.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:08 smithi189 bash[23614]: cephadm 2024-03-07T03:17:07.750213+0000 mgr.y (mgr.34107) 87 : cephadm 1 Deploying daemon mon.b on smithi189 2024-03-07T03:17:08.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[79878]: cluster 2024-03-07T03:17:07.381976+0000 mon.a (mon.0) 278 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2024-03-07T03:17:08.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[79878]: cephadm 2024-03-07T03:17:07.733829+0000 mgr.y (mgr.34107) 86 : cephadm 1 Upgrade: Updating mon.b 2024-03-07T03:17:08.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[79878]: audit 2024-03-07T03:17:07.741237+0000 mon.a (mon.0) 279 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:08.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[79878]: audit 2024-03-07T03:17:07.745459+0000 mon.a (mon.0) 280 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:17:08.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[79878]: audit 2024-03-07T03:17:07.746917+0000 mon.a (mon.0) 281 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:17:08.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[79878]: audit 2024-03-07T03:17:07.748328+0000 mon.a (mon.0) 282 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:08.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[79878]: cephadm 2024-03-07T03:17:07.750213+0000 mgr.y (mgr.34107) 87 : cephadm 1 Deploying daemon mon.b on smithi189 2024-03-07T03:17:08.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[82329]: cluster 2024-03-07T03:17:07.381976+0000 mon.a (mon.0) 278 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2024-03-07T03:17:08.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[82329]: cephadm 2024-03-07T03:17:07.733829+0000 mgr.y (mgr.34107) 86 : cephadm 1 Upgrade: Updating mon.b 2024-03-07T03:17:08.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[82329]: audit 2024-03-07T03:17:07.741237+0000 mon.a (mon.0) 279 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:08.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[82329]: audit 2024-03-07T03:17:07.745459+0000 mon.a (mon.0) 280 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:17:08.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[82329]: audit 2024-03-07T03:17:07.746917+0000 mon.a (mon.0) 281 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:17:08.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[82329]: audit 2024-03-07T03:17:07.748328+0000 mon.a (mon.0) 282 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:08.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:08 smithi145 bash[82329]: cephadm 2024-03-07T03:17:07.750213+0000 mgr.y (mgr.34107) 87 : cephadm 1 Deploying daemon mon.b on smithi189 2024-03-07T03:17:08.934 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:17:09.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:09 smithi189 bash[23614]: cluster 2024-03-07T03:17:08.385075+0000 mon.a (mon.0) 283 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2024-03-07T03:17:09.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:09 smithi189 bash[23614]: cluster 2024-03-07T03:17:08.569726+0000 mgr.y (mgr.34107) 88 : cluster 0 pgmap v172: 105 pgs: 105 active+clean; 49 MiB data, 520 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-03-07T03:17:09.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:09 smithi145 bash[79878]: cluster 2024-03-07T03:17:08.385075+0000 mon.a (mon.0) 283 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2024-03-07T03:17:09.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:09 smithi145 bash[79878]: cluster 2024-03-07T03:17:08.569726+0000 mgr.y (mgr.34107) 88 : cluster 0 pgmap v172: 105 pgs: 105 active+clean; 49 MiB data, 520 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-03-07T03:17:09.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:09 smithi145 bash[82329]: cluster 2024-03-07T03:17:08.385075+0000 mon.a (mon.0) 283 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2024-03-07T03:17:09.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:09 smithi145 bash[82329]: cluster 2024-03-07T03:17:08.569726+0000 mgr.y (mgr.34107) 88 : cluster 0 pgmap v172: 105 pgs: 105 active+clean; 49 MiB data, 520 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-03-07T03:17:09.878 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.Snaps (4029 ms) 2024-03-07T03:17:09.878 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.SnapDiscard 2024-03-07T03:17:09.932 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:09 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:09.932 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:17:09 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:09.933 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:17:09 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:09.933 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:17:09 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:09.933 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:17:09 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:09.934 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:09 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:09.934 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:17:09 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:09.934 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:17:09 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:09.935 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:17:09 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:10.190 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:09 smithi189 systemd[1]: Stopping Ceph mon.b for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:17:10.190 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:10 smithi189 bash[23614]: debug 2024-03-07T03:17:10.061+0000 7f5f201a1700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T03:17:10.190 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:10 smithi189 bash[23614]: debug 2024-03-07T03:17:10.061+0000 7f5f201a1700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-03-07T03:17:11.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:10 smithi189 bash[66851]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mon-b 2024-03-07T03:17:11.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:10 smithi189 bash[66913]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mon-b 2024-03-07T03:17:11.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:10 smithi189 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.b.service: Deactivated successfully. 2024-03-07T03:17:11.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:10 smithi189 systemd[1]: Stopped Ceph mon.b for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:17:11.580 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:17:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:11.580 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:17:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:11.580 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:17:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:11.581 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:17:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:11.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:11.581 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:17:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:11.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:11.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:11 smithi189 systemd[1]: Started Ceph mon.b for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:17:11.582 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:17:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:11.582 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:17:11 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:12.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[59814]: ts=2024-03-07T03:17:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:12.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.153+0000 7f558080fc80 0 set uid:gid to 167:167 (ceph:ceph) 2024-03-07T03:17:12.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.153+0000 7f558080fc80 0 ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev), process ceph-mon, pid 7 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.153+0000 7f558080fc80 0 pidfile_write: ignore empty --pid-file 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 0 load: jerasure load: lrc 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: RocksDB version: 7.9.2 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Git sha 0 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Compile date 2024-03-06 19:59:24 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: DB SUMMARY 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: DB Session ID: QSGI1GLQJFG5KZULM326 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: CURRENT file: CURRENT 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: IDENTITY file: IDENTITY 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: MANIFEST file: MANIFEST-000010 size: 2630 Bytes 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000042.sst 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000040.log size: 4095121 ; 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.error_if_exists: 0 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.create_if_missing: 0 2024-03-07T03:17:12.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.paranoid_checks: 1 2024-03-07T03:17:12.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-03-07T03:17:12.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-07T03:17:12.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-07T03:17:12.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.env: 0x55daf8f24b20 2024-03-07T03:17:12.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.fs: PosixFileSystem 2024-03-07T03:17:12.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.info_log: 0x55daf99c5920 2024-03-07T03:17:12.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_file_opening_threads: 16 2024-03-07T03:17:12.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.statistics: (nil) 2024-03-07T03:17:12.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.use_fsync: 0 2024-03-07T03:17:12.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_log_file_size: 0 2024-03-07T03:17:12.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-07T03:17:12.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-03-07T03:17:12.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.keep_log_file_num: 1000 2024-03-07T03:17:12.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.recycle_log_file_num: 0 2024-03-07T03:17:12.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.allow_fallocate: 1 2024-03-07T03:17:12.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.allow_mmap_reads: 0 2024-03-07T03:17:12.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.allow_mmap_writes: 0 2024-03-07T03:17:12.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.use_direct_reads: 0 2024-03-07T03:17:12.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-07T03:17:12.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.create_missing_column_families: 0 2024-03-07T03:17:12.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.db_log_dir: 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.wal_dir: 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.advise_random_on_open: 1 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.db_write_buffer_size: 0 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.write_buffer_manager: 0x55daf9998dc0 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-03-07T03:17:12.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.rate_limiter: (nil) 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.wal_recovery_mode: 2 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.enable_thread_tracking: 0 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.enable_pipelined_write: 0 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.unordered_write: 0 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.row_cache: None 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.wal_filter: None 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.allow_ingest_behind: 0 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.two_write_queues: 0 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.manual_wal_flush: 0 2024-03-07T03:17:12.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.wal_compression: 0 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.atomic_flush: 0 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.log_readahead_size: 0 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.best_efforts_recovery: 0 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.allow_data_in_errors: 0 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.db_host_id: __hostname__ 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_background_jobs: 2 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_background_compactions: -1 2024-03-07T03:17:12.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_subcompactions: 1 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_total_wal_size: 0 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_open_files: -1 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.bytes_per_sync: 0 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compaction_readahead_size: 0 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_background_flushes: -1 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Compression algorithms supported: 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-07T03:17:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: kZSTD supported: 0 2024-03-07T03:17:12.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: kXpressCompression supported: 0 2024-03-07T03:17:12.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: kLZ4HCCompression supported: 1 2024-03-07T03:17:12.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: kZlibCompression supported: 1 2024-03-07T03:17:12.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: kSnappyCompression supported: 1 2024-03-07T03:17:12.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: kLZ4Compression supported: 1 2024-03-07T03:17:12.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: kBZip2Compression supported: 0 2024-03-07T03:17:12.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-03-07T03:17:12.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-03-07T03:17:12.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 2024-03-07T03:17:12.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-07T03:17:12.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.merge_operator: 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compaction_filter: None 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compaction_filter_factory: None 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.sst_partitioner_factory: None 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55daf99c5a20) 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: cache_index_and_filter_blocks: 1 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: pin_top_level_index_and_filter: 1 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: index_type: 0 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: data_block_index_type: 0 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: index_shortening: 1 2024-03-07T03:17:12.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: data_block_hash_table_util_ratio: 0.750000 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: checksum: 4 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: no_block_cache: 0 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: block_cache: 0x55daf99c9090 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: block_cache_name: BinnedLRUCache 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: block_cache_options: 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: capacity : 536870912 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: num_shard_bits : 4 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: strict_capacity_limit : 0 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: high_pri_pool_ratio: 0.000 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: block_cache_compressed: (nil) 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: persistent_cache: (nil) 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: block_size: 4096 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: block_size_deviation: 10 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: block_restart_interval: 16 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: index_block_restart_interval: 1 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: metadata_block_size: 4096 2024-03-07T03:17:12.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: partition_filters: 0 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: use_delta_encoding: 1 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: filter_policy: bloomfilter 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: whole_key_filtering: 1 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: verify_compression: 0 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: read_amp_bytes_per_bit: 0 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: format_version: 5 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: enable_index_compression: 1 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: block_align: 0 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: max_auto_readahead_size: 262144 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: prepopulate_block_cache: 0 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: initial_auto_readahead_size: 8192 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: num_file_reads_for_auto_readahead: 2 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.write_buffer_size: 33554432 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_write_buffer_number: 2 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compression: NoCompression 2024-03-07T03:17:12.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.bottommost_compression: Disabled 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.prefix_extractor: nullptr 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.num_levels: 7 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compression_opts.level: 32767 2024-03-07T03:17:12.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compression_opts.strategy: 0 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compression_opts.enabled: false 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.target_file_size_base: 67108864 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-07T03:17:12.594 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.arena_block_size: 1048576 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.disable_auto_compactions: 0 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-07T03:17:12.595 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.table_properties_collectors: 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.inplace_update_support: 0 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.bloom_locality: 0 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.max_successive_merges: 0 2024-03-07T03:17:12.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.paranoid_file_checks: 0 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.force_consistency_checks: 1 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.report_bg_io_stats: 0 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.ttl: 2592000 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.enable_blob_files: false 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.min_blob_size: 0 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.blob_file_size: 268435456 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-07T03:17:12.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.blob_file_starting_level: 0 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 44, last_sequence is 24279, log_number is 40,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 40 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 40 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 9aaa9f57-eed6-4d51-b5b7-c1373ff5ef78 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781432167739, "job": 1, "event": "recovery_started", "wal_files": [40]} 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.161+0000 7f558080fc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #40 mode 2 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.173+0000 7f558080fc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781432180532, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 45, "file_size": 2463132, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 24284, "largest_seqno": 25971, "table_properties": {"data_size": 2454891, "index_size": 5622, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1605, "raw_key_size": 13937, "raw_average_key_size": 22, "raw_value_size": 2440112, "raw_average_value_size": 3910, "num_data_blocks": 244, "num_entries": 624, "num_filter_entries": 624, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1709781432, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "9aaa9f57-eed6-4d51-b5b7-c1373ff5ef78", "db_session_id": "QSGI1GLQJFG5KZULM326", "orig_file_number": 45, "seqno_to_time_mapping": "N/A"}} 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.173+0000 7f558080fc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781432180652, "job": 1, "event": "recovery_finished"} 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.173+0000 7f558080fc80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 47 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.173+0000 7f558080fc80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-07T03:17:12.598 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.177+0000 7f558080fc80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000040.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.177+0000 7f558080fc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55daf9ac0000 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.177+0000 7f558080fc80 4 rocksdb: DB pointer 0x55daf9aaa000 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.177+0000 7f556ef78700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.177+0000 7f556ef78700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: ** DB Stats ** 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: ** Compaction Stats [default] ** 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-07T03:17:12.599 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: L0 1/0 2.35 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 517.5 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: L6 1/0 11.77 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Sum 2/0 14.12 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 517.5 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 517.5 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: ** Compaction Stats [default] ** 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 517.5 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Flush(GB): cumulative 0.002, interval 0.002 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: AddFile(Total Files): cumulative 0, interval 0 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: AddFile(L0 Files): cumulative 0, interval 0 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: AddFile(Keys): cumulative 0, interval 0 2024-03-07T03:17:12.600 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Cumulative compaction: 0.00 GB write, 146.25 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Interval compaction: 0.00 GB write, 146.25 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Block cache BinnedLRUCache@0x55daf99c9090#7 capacity: 512.00 MB usage: 7.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: ** File Read Latency Histogram By Level [default] ** 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.177+0000 7f558080fc80 0 starting mon.b rank 1 at public addrs [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] at bind addrs [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.177+0000 7f558080fc80 1 mon.b@-1(???) e3 preinit fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.181+0000 7f558080fc80 0 mon.b@-1(???).mds e10 new map 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.181+0000 7f558080fc80 0 mon.b@-1(???).mds e10 print_map 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: e10 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: enable_multiple, ever_enabled_multiple: 1,1 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: legacy client fscid: 1 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Filesystem 'foo' (1) 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: fs_name foo 2024-03-07T03:17:12.601 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: epoch 10 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: created 2024-03-07T03:08:49.405527+0000 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: modified 2024-03-07T03:09:00.347215+0000 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: tableserver 0 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: root 0 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: session_timeout 60 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: session_autoclose 300 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: max_file_size 1099511627776 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: max_xattr_size 65536 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: required_client_features {} 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: last_failure 0 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: last_failure_osd_epoch 70 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: max_mds 1 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: in 0 2024-03-07T03:17:12.602 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: up {0=24376} 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: failed 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: damaged 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: stopped 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: data_pools [4] 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: metadata_pool 3 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: inline_data disabled 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: balancer 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: bal_rank_mask -1 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: standby_count_wanted 1 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: [mds.foo.smithi189.zkinbe{0:24376} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.189:6832/1682575491,v1:172.21.15.189:6833/1682575491] compat {c=[1],r=[1],i=[7ff]}] 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: Standby daemons: 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: 2024-03-07T03:17:12.603 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: [mds.foo.smithi145.zmgvgw{-1:14589} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.145:6834/4165785656,v1:172.21.15.145:6835/4165785656] compat {c=[1],r=[1],i=[7ff]}] 2024-03-07T03:17:12.604 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.181+0000 7f558080fc80 0 mon.b@-1(???).osd e389 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-07T03:17:12.604 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.181+0000 7f558080fc80 0 mon.b@-1(???).osd e389 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:17:12.604 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.181+0000 7f558080fc80 0 mon.b@-1(???).osd e389 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:17:12.604 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.181+0000 7f558080fc80 0 mon.b@-1(???).osd e389 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:17:12.604 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:12 smithi189 bash[66969]: debug 2024-03-07T03:17:12.181+0000 7f558080fc80 1 mon.b@-1(???).paxosservice(auth 1..23) refresh upgraded, format 0 -> 3 2024-03-07T03:17:13.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:17:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:17:12] "GET /metrics HTTP/1.1" 200 36244 "" "Prometheus/2.43.0" 2024-03-07T03:17:13.536 INFO:teuthology.orchestra.run.smithi145.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:17:13.772 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[82329]: cluster 2024-03-07T03:17:09.409444+0000 mon.a (mon.0) 284 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2024-03-07T03:17:13.772 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: cluster 2024-03-07T03:17:09.409444+0000 mon.a (mon.0) 284 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2024-03-07T03:17:13.772 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.482+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.482+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.482+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.482+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.482+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.482+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.482+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.482+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.482+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.482+0000 7fdb44ed0700 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.506+0000 7fdb476d5700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.506+0000 7fdb476d5700 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.506+0000 7fdb476d5700 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.506+0000 7fdb476d5700 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.773 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.506+0000 7fdb476d5700 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.774 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.506+0000 7fdb476d5700 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.774 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.506+0000 7fdb476d5700 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.774 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.506+0000 7fdb476d5700 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.774 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.506+0000 7fdb476d5700 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.774 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:13 smithi145 bash[79878]: debug 2024-03-07T03:17:13.506+0000 7fdb476d5700 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:13.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:13 smithi189 bash[66969]: cluster 2024-03-07T03:17:09.409444+0000 mon.a (mon.0) 284 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2024-03-07T03:17:13.830 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:13 smithi189 bash[59814]: ts=2024-03-07T03:17:13.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:14.305 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'sleep 60' 2024-03-07T03:17:14.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.488536+0000 mon.a (mon.0) 295 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.488574+0000 mon.a (mon.0) 296 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.488612+0000 mon.a (mon.0) 297 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.488648+0000 mon.a (mon.0) 298 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.488684+0000 mon.a (mon.0) 299 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.488722+0000 mon.a (mon.0) 300 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.500007+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.500491+0000 mon.a (mon.0) 301 : cluster 1 mon.a calling monitor election 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.500512+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: audit 2024-03-07T03:17:13.502870+0000 mon.a (mon.0) 302 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: audit 2024-03-07T03:17:13.503245+0000 mon.a (mon.0) 303 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: audit 2024-03-07T03:17:13.503693+0000 mon.a (mon.0) 304 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.504501+0000 mon.a (mon.0) 305 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.510567+0000 mon.a (mon.0) 306 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:17:14.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.511594+0000 mon.a (mon.0) 307 : cluster 0 fsmap foo:1 {0=foo.smithi189.zkinbe=up:active} 1 up:standby 2024-03-07T03:17:14.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.511671+0000 mon.a (mon.0) 308 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2024-03-07T03:17:14.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.512450+0000 mon.a (mon.0) 309 : cluster 0 mgrmap e38: y(active, since 2m), standbys: x 2024-03-07T03:17:14.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.512745+0000 mon.a (mon.0) 310 : cluster 4 Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:17:14.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.512795+0000 mon.a (mon.0) 311 : cluster 4 [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:17:14.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.512836+0000 mon.a (mon.0) 312 : cluster 4 osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.512881+0000 mon.a (mon.0) 313 : cluster 4 osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.488536+0000 mon.a (mon.0) 295 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.488574+0000 mon.a (mon.0) 296 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.488612+0000 mon.a (mon.0) 297 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.488648+0000 mon.a (mon.0) 298 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.488684+0000 mon.a (mon.0) 299 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.488722+0000 mon.a (mon.0) 300 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.500007+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.500491+0000 mon.a (mon.0) 301 : cluster 1 mon.a calling monitor election 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.500512+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: audit 2024-03-07T03:17:13.502870+0000 mon.a (mon.0) 302 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: audit 2024-03-07T03:17:13.503245+0000 mon.a (mon.0) 303 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: audit 2024-03-07T03:17:13.503693+0000 mon.a (mon.0) 304 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.504501+0000 mon.a (mon.0) 305 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.510567+0000 mon.a (mon.0) 306 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.511594+0000 mon.a (mon.0) 307 : cluster 0 fsmap foo:1 {0=foo.smithi189.zkinbe=up:active} 1 up:standby 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.511671+0000 mon.a (mon.0) 308 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2024-03-07T03:17:14.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.512450+0000 mon.a (mon.0) 309 : cluster 0 mgrmap e38: y(active, since 2m), standbys: x 2024-03-07T03:17:14.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.512745+0000 mon.a (mon.0) 310 : cluster 4 Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:17:14.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.512795+0000 mon.a (mon.0) 311 : cluster 4 [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:17:14.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.512836+0000 mon.a (mon.0) 312 : cluster 4 osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.512881+0000 mon.a (mon.0) 313 : cluster 4 osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.512928+0000 mon.a (mon.0) 314 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.512949+0000 mon.a (mon.0) 315 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.512992+0000 mon.a (mon.0) 316 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.513038+0000 mon.a (mon.0) 317 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.513070+0000 mon.a (mon.0) 318 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: cluster 2024-03-07T03:17:13.513111+0000 mon.a (mon.0) 319 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: audit 2024-03-07T03:17:13.531935+0000 mon.a (mon.0) 320 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: audit 2024-03-07T03:17:13.540654+0000 mon.a (mon.0) 321 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: audit 2024-03-07T03:17:13.547627+0000 mon.a (mon.0) 322 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: audit 2024-03-07T03:17:13.556522+0000 mon.a (mon.0) 323 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: audit 2024-03-07T03:17:13.578506+0000 mon.a (mon.0) 324 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:14.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: audit 2024-03-07T03:17:14.050004+0000 mon.a (mon.0) 325 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[82329]: audit 2024-03-07T03:17:14.056449+0000 mon.a (mon.0) 326 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.512928+0000 mon.a (mon.0) 314 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.512949+0000 mon.a (mon.0) 315 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.512992+0000 mon.a (mon.0) 316 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.564 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.513038+0000 mon.a (mon.0) 317 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.513070+0000 mon.a (mon.0) 318 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: cluster 2024-03-07T03:17:13.513111+0000 mon.a (mon.0) 319 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: audit 2024-03-07T03:17:13.531935+0000 mon.a (mon.0) 320 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: audit 2024-03-07T03:17:13.540654+0000 mon.a (mon.0) 321 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: audit 2024-03-07T03:17:13.547627+0000 mon.a (mon.0) 322 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: audit 2024-03-07T03:17:13.556522+0000 mon.a (mon.0) 323 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: audit 2024-03-07T03:17:13.578506+0000 mon.a (mon.0) 324 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:14.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: audit 2024-03-07T03:17:14.050004+0000 mon.a (mon.0) 325 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:14 smithi145 bash[79878]: audit 2024-03-07T03:17:14.056449+0000 mon.a (mon.0) 326 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.488536+0000 mon.a (mon.0) 295 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.488574+0000 mon.a (mon.0) 296 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.488612+0000 mon.a (mon.0) 297 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.488648+0000 mon.a (mon.0) 298 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.488684+0000 mon.a (mon.0) 299 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.488722+0000 mon.a (mon.0) 300 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.500007+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-03-07T03:17:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.500491+0000 mon.a (mon.0) 301 : cluster 1 mon.a calling monitor election 2024-03-07T03:17:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.500512+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2024-03-07T03:17:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: audit 2024-03-07T03:17:13.502870+0000 mon.a (mon.0) 302 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:17:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: audit 2024-03-07T03:17:13.503245+0000 mon.a (mon.0) 303 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:17:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: audit 2024-03-07T03:17:13.503693+0000 mon.a (mon.0) 304 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:17:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.504501+0000 mon.a (mon.0) 305 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:17:14.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.510567+0000 mon.a (mon.0) 306 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.145:3300/0,v1:172.21.15.145:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.145:3301/0,v1:172.21.15.145:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:17:14.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.511594+0000 mon.a (mon.0) 307 : cluster 0 fsmap foo:1 {0=foo.smithi189.zkinbe=up:active} 1 up:standby 2024-03-07T03:17:14.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.511671+0000 mon.a (mon.0) 308 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2024-03-07T03:17:14.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.512450+0000 mon.a (mon.0) 309 : cluster 0 mgrmap e38: y(active, since 2m), standbys: x 2024-03-07T03:17:14.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.512745+0000 mon.a (mon.0) 310 : cluster 4 Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:17:14.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.512795+0000 mon.a (mon.0) 311 : cluster 4 [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:17:14.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.512836+0000 mon.a (mon.0) 312 : cluster 4 osd.0's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.512881+0000 mon.a (mon.0) 313 : cluster 4 osd.1's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.512928+0000 mon.a (mon.0) 314 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.512949+0000 mon.a (mon.0) 315 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.512992+0000 mon.a (mon.0) 316 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.513038+0000 mon.a (mon.0) 317 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.513070+0000 mon.a (mon.0) 318 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: cluster 2024-03-07T03:17:13.513111+0000 mon.a (mon.0) 319 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.1/32,172.21.0.0/20,172.21.0.2/32' subnet 2024-03-07T03:17:14.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: audit 2024-03-07T03:17:13.531935+0000 mon.a (mon.0) 320 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: audit 2024-03-07T03:17:13.540654+0000 mon.a (mon.0) 321 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: audit 2024-03-07T03:17:13.547627+0000 mon.a (mon.0) 322 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: audit 2024-03-07T03:17:13.556522+0000 mon.a (mon.0) 323 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: audit 2024-03-07T03:17:13.578506+0000 mon.a (mon.0) 324 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:14.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: audit 2024-03-07T03:17:14.050004+0000 mon.a (mon.0) 325 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:14.833 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:14 smithi189 bash[66969]: audit 2024-03-07T03:17:14.056449+0000 mon.a (mon.0) 326 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:15.721 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.SnapDiscard (5843 ms) 2024-03-07T03:17:15.722 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.CloneDiscard 2024-03-07T03:17:15.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:15 smithi145 bash[79878]: cluster 2024-03-07T03:17:14.506591+0000 mon.a (mon.0) 327 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2024-03-07T03:17:15.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:15 smithi145 bash[79878]: cluster 2024-03-07T03:17:14.571807+0000 mgr.y (mgr.34107) 93 : cluster 0 pgmap v177: 105 pgs: 105 active+clean; 65 MiB data, 569 MiB used, 715 GiB / 715 GiB avail; 23 MiB/s rd, 10 MiB/s wr, 161 op/s 2024-03-07T03:17:15.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:15 smithi145 bash[82329]: cluster 2024-03-07T03:17:14.506591+0000 mon.a (mon.0) 327 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2024-03-07T03:17:15.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:15 smithi145 bash[82329]: cluster 2024-03-07T03:17:14.571807+0000 mgr.y (mgr.34107) 93 : cluster 0 pgmap v177: 105 pgs: 105 active+clean; 65 MiB data, 569 MiB used, 715 GiB / 715 GiB avail; 23 MiB/s rd, 10 MiB/s wr, 161 op/s 2024-03-07T03:17:15.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:15 smithi189 bash[66969]: cluster 2024-03-07T03:17:14.506591+0000 mon.a (mon.0) 327 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2024-03-07T03:17:15.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:15 smithi189 bash[66969]: cluster 2024-03-07T03:17:14.571807+0000 mgr.y (mgr.34107) 93 : cluster 0 pgmap v177: 105 pgs: 105 active+clean; 65 MiB data, 569 MiB used, 715 GiB / 715 GiB avail; 23 MiB/s rd, 10 MiB/s wr, 161 op/s 2024-03-07T03:17:16.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:16 smithi145 bash[79878]: cluster 2024-03-07T03:17:15.516347+0000 mon.a (mon.0) 328 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2024-03-07T03:17:16.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:16 smithi145 bash[82329]: cluster 2024-03-07T03:17:15.516347+0000 mon.a (mon.0) 328 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2024-03-07T03:17:16.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:16 smithi189 bash[66969]: cluster 2024-03-07T03:17:15.516347+0000 mon.a (mon.0) 328 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2024-03-07T03:17:16.892 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.CloneDiscard (1171 ms) 2024-03-07T03:17:16.892 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.CloneShrink 2024-03-07T03:17:17.527 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:17:17 smithi145 bash[72238]: debug 2024-03-07T03:17:17.186+0000 7f318c55a700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-03-07T03:17:17.706 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.CloneShrink (814 ms) 2024-03-07T03:17:17.707 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.CloneExpand 2024-03-07T03:17:17.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:17 smithi145 bash[82329]: cluster 2024-03-07T03:17:16.535854+0000 mon.a (mon.0) 329 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2024-03-07T03:17:17.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:17 smithi145 bash[82329]: cluster 2024-03-07T03:17:16.572324+0000 mgr.y (mgr.34107) 94 : cluster 0 pgmap v180: 105 pgs: 105 active+clean; 65 MiB data, 569 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 2.7 MiB/s wr, 48 op/s 2024-03-07T03:17:17.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:17 smithi145 bash[82329]: cluster 2024-03-07T03:17:17.274524+0000 mon.a (mon.0) 330 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2024-03-07T03:17:17.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:17 smithi145 bash[79878]: cluster 2024-03-07T03:17:16.535854+0000 mon.a (mon.0) 329 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2024-03-07T03:17:17.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:17 smithi145 bash[79878]: cluster 2024-03-07T03:17:16.572324+0000 mgr.y (mgr.34107) 94 : cluster 0 pgmap v180: 105 pgs: 105 active+clean; 65 MiB data, 569 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 2.7 MiB/s wr, 48 op/s 2024-03-07T03:17:17.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:17 smithi145 bash[79878]: cluster 2024-03-07T03:17:17.274524+0000 mon.a (mon.0) 330 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2024-03-07T03:17:17.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:17 smithi189 bash[66969]: cluster 2024-03-07T03:17:16.535854+0000 mon.a (mon.0) 329 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2024-03-07T03:17:17.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:17 smithi189 bash[66969]: cluster 2024-03-07T03:17:16.572324+0000 mgr.y (mgr.34107) 94 : cluster 0 pgmap v180: 105 pgs: 105 active+clean; 65 MiB data, 569 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 2.7 MiB/s wr, 48 op/s 2024-03-07T03:17:17.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:17 smithi189 bash[66969]: cluster 2024-03-07T03:17:17.274524+0000 mon.a (mon.0) 330 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2024-03-07T03:17:18.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:18 smithi145 bash[79878]: audit 2024-03-07T03:17:17.631915+0000 mon.a (mon.0) 331 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:18.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:18 smithi145 bash[82329]: audit 2024-03-07T03:17:17.631915+0000 mon.a (mon.0) 331 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:18.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:18 smithi189 bash[66969]: audit 2024-03-07T03:17:17.631915+0000 mon.a (mon.0) 331 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:18.997 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.CloneExpand (1290 ms) 2024-03-07T03:17:18.997 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.CloneHideParent 2024-03-07T03:17:19.049 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:17:19.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:19 smithi145 bash[79878]: cluster 2024-03-07T03:17:18.563148+0000 mon.a (mon.0) 332 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2024-03-07T03:17:19.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:19 smithi145 bash[79878]: cluster 2024-03-07T03:17:18.572970+0000 mgr.y (mgr.34107) 95 : cluster 0 pgmap v183: 105 pgs: 105 active+clean; 65 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 264 KiB/s rd, 25 KiB/s wr, 364 op/s 2024-03-07T03:17:19.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:19 smithi145 bash[82329]: cluster 2024-03-07T03:17:18.563148+0000 mon.a (mon.0) 332 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2024-03-07T03:17:19.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:19 smithi145 bash[82329]: cluster 2024-03-07T03:17:18.572970+0000 mgr.y (mgr.34107) 95 : cluster 0 pgmap v183: 105 pgs: 105 active+clean; 65 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 264 KiB/s rd, 25 KiB/s wr, 364 op/s 2024-03-07T03:17:19.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:19 smithi189 bash[66969]: cluster 2024-03-07T03:17:18.563148+0000 mon.a (mon.0) 332 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2024-03-07T03:17:19.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:19 smithi189 bash[66969]: cluster 2024-03-07T03:17:18.572970+0000 mgr.y (mgr.34107) 95 : cluster 0 pgmap v183: 105 pgs: 105 active+clean; 65 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 264 KiB/s rd, 25 KiB/s wr, 364 op/s 2024-03-07T03:17:20.554 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:17:20 smithi189 bash[49584]: logger=cleanup t=2024-03-07T03:17:20.228508097Z level=info msg="Completed cleanup jobs" duration=79.653369ms 2024-03-07T03:17:20.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:20 smithi145 bash[82329]: cluster 2024-03-07T03:17:19.575834+0000 mon.a (mon.0) 333 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2024-03-07T03:17:20.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:20 smithi145 bash[82329]: audit 2024-03-07T03:17:19.587689+0000 mon.a (mon.0) 334 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:20.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:20 smithi145 bash[82329]: audit 2024-03-07T03:17:19.603411+0000 mon.a (mon.0) 335 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:20.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:20 smithi145 bash[79878]: cluster 2024-03-07T03:17:19.575834+0000 mon.a (mon.0) 333 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2024-03-07T03:17:20.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:20 smithi145 bash[79878]: audit 2024-03-07T03:17:19.587689+0000 mon.a (mon.0) 334 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:20.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:20 smithi145 bash[79878]: audit 2024-03-07T03:17:19.603411+0000 mon.a (mon.0) 335 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:20 smithi189 bash[66969]: cluster 2024-03-07T03:17:19.575834+0000 mon.a (mon.0) 333 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2024-03-07T03:17:20.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:20 smithi189 bash[66969]: audit 2024-03-07T03:17:19.587689+0000 mon.a (mon.0) 334 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:20.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:20 smithi189 bash[66969]: audit 2024-03-07T03:17:19.603411+0000 mon.a (mon.0) 335 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:22.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:21 smithi145 bash[79878]: cluster 2024-03-07T03:17:20.573638+0000 mgr.y (mgr.34107) 96 : cluster 0 pgmap v185: 105 pgs: 105 active+clean; 65 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 261 KiB/s rd, 24 KiB/s wr, 360 op/s 2024-03-07T03:17:22.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:21 smithi145 bash[79878]: cluster 2024-03-07T03:17:20.604463+0000 mon.a (mon.0) 336 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2024-03-07T03:17:22.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:21 smithi145 bash[79878]: audit 2024-03-07T03:17:20.761458+0000 mon.a (mon.0) 337 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:22.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:21 smithi145 bash[79878]: audit 2024-03-07T03:17:20.772892+0000 mon.a (mon.0) 338 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:22.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:21 smithi145 bash[82329]: cluster 2024-03-07T03:17:20.573638+0000 mgr.y (mgr.34107) 96 : cluster 0 pgmap v185: 105 pgs: 105 active+clean; 65 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 261 KiB/s rd, 24 KiB/s wr, 360 op/s 2024-03-07T03:17:22.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:21 smithi145 bash[82329]: cluster 2024-03-07T03:17:20.604463+0000 mon.a (mon.0) 336 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2024-03-07T03:17:22.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:21 smithi145 bash[82329]: audit 2024-03-07T03:17:20.761458+0000 mon.a (mon.0) 337 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:22.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:21 smithi145 bash[82329]: audit 2024-03-07T03:17:20.772892+0000 mon.a (mon.0) 338 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:22.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:21 smithi189 bash[66969]: cluster 2024-03-07T03:17:20.573638+0000 mgr.y (mgr.34107) 96 : cluster 0 pgmap v185: 105 pgs: 105 active+clean; 65 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 261 KiB/s rd, 24 KiB/s wr, 360 op/s 2024-03-07T03:17:22.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:21 smithi189 bash[66969]: cluster 2024-03-07T03:17:20.604463+0000 mon.a (mon.0) 336 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2024-03-07T03:17:22.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:21 smithi189 bash[66969]: audit 2024-03-07T03:17:20.761458+0000 mon.a (mon.0) 337 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:22.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:21 smithi189 bash[66969]: audit 2024-03-07T03:17:20.772892+0000 mon.a (mon.0) 338 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:22.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:22 smithi189 bash[59814]: ts=2024-03-07T03:17:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:23.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:22 smithi145 bash[79878]: cluster 2024-03-07T03:17:21.615350+0000 mon.a (mon.0) 339 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2024-03-07T03:17:23.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:17:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:17:22] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.43.0" 2024-03-07T03:17:23.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:22 smithi145 bash[82329]: cluster 2024-03-07T03:17:21.615350+0000 mon.a (mon.0) 339 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2024-03-07T03:17:23.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:22 smithi189 bash[66969]: cluster 2024-03-07T03:17:21.615350+0000 mon.a (mon.0) 339 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2024-03-07T03:17:23.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:23 smithi189 bash[66969]: cluster 2024-03-07T03:17:22.574363+0000 mgr.y (mgr.34107) 97 : cluster 0 pgmap v188: 105 pgs: 105 active+clean; 65 MiB data, 591 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:23.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:23 smithi189 bash[66969]: cluster 2024-03-07T03:17:22.628674+0000 mon.a (mon.0) 340 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2024-03-07T03:17:23.830 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:23 smithi189 bash[59814]: ts=2024-03-07T03:17:23.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:24.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:23 smithi145 bash[79878]: cluster 2024-03-07T03:17:22.574363+0000 mgr.y (mgr.34107) 97 : cluster 0 pgmap v188: 105 pgs: 105 active+clean; 65 MiB data, 591 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:24.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:23 smithi145 bash[79878]: cluster 2024-03-07T03:17:22.628674+0000 mon.a (mon.0) 340 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2024-03-07T03:17:24.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:23 smithi145 bash[82329]: cluster 2024-03-07T03:17:22.574363+0000 mgr.y (mgr.34107) 97 : cluster 0 pgmap v188: 105 pgs: 105 active+clean; 65 MiB data, 591 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:24.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:23 smithi145 bash[82329]: cluster 2024-03-07T03:17:22.628674+0000 mon.a (mon.0) 340 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2024-03-07T03:17:25.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:24 smithi145 bash[79878]: cluster 2024-03-07T03:17:23.633626+0000 mon.a (mon.0) 341 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2024-03-07T03:17:25.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:24 smithi145 bash[82329]: cluster 2024-03-07T03:17:23.633626+0000 mon.a (mon.0) 341 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2024-03-07T03:17:25.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:24 smithi189 bash[66969]: cluster 2024-03-07T03:17:23.633626+0000 mon.a (mon.0) 341 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2024-03-07T03:17:25.901 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.CloneHideParent (6904 ms) 2024-03-07T03:17:25.901 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Clone 2024-03-07T03:17:26.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:25 smithi145 bash[79878]: cluster 2024-03-07T03:17:24.575004+0000 mgr.y (mgr.34107) 98 : cluster 0 pgmap v191: 105 pgs: 105 active+clean; 65 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-07T03:17:26.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:25 smithi145 bash[79878]: cluster 2024-03-07T03:17:24.657106+0000 mon.a (mon.0) 342 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2024-03-07T03:17:26.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:25 smithi145 bash[82329]: cluster 2024-03-07T03:17:24.575004+0000 mgr.y (mgr.34107) 98 : cluster 0 pgmap v191: 105 pgs: 105 active+clean; 65 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-07T03:17:26.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:25 smithi145 bash[82329]: cluster 2024-03-07T03:17:24.657106+0000 mon.a (mon.0) 342 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2024-03-07T03:17:26.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:25 smithi189 bash[66969]: cluster 2024-03-07T03:17:24.575004+0000 mgr.y (mgr.34107) 98 : cluster 0 pgmap v191: 105 pgs: 105 active+clean; 65 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-07T03:17:26.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:25 smithi189 bash[66969]: cluster 2024-03-07T03:17:24.657106+0000 mon.a (mon.0) 342 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2024-03-07T03:17:26.910 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:26 smithi189 bash[66969]: cluster 2024-03-07T03:17:25.657438+0000 mon.a (mon.0) 343 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2024-03-07T03:17:27.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:26 smithi145 bash[79878]: cluster 2024-03-07T03:17:25.657438+0000 mon.a (mon.0) 343 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2024-03-07T03:17:27.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:26 smithi145 bash[82329]: cluster 2024-03-07T03:17:25.657438+0000 mon.a (mon.0) 343 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2024-03-07T03:17:28.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:27 smithi145 bash[79878]: cluster 2024-03-07T03:17:26.575716+0000 mgr.y (mgr.34107) 99 : cluster 0 pgmap v194: 105 pgs: 105 active+clean; 65 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-07T03:17:28.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:27 smithi145 bash[79878]: cluster 2024-03-07T03:17:26.677782+0000 mon.a (mon.0) 344 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2024-03-07T03:17:28.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:27 smithi145 bash[82329]: cluster 2024-03-07T03:17:26.575716+0000 mgr.y (mgr.34107) 99 : cluster 0 pgmap v194: 105 pgs: 105 active+clean; 65 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-07T03:17:28.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:27 smithi145 bash[82329]: cluster 2024-03-07T03:17:26.677782+0000 mon.a (mon.0) 344 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2024-03-07T03:17:28.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:27 smithi189 bash[66969]: cluster 2024-03-07T03:17:26.575716+0000 mgr.y (mgr.34107) 99 : cluster 0 pgmap v194: 105 pgs: 105 active+clean; 65 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-07T03:17:28.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:27 smithi189 bash[66969]: cluster 2024-03-07T03:17:26.677782+0000 mon.a (mon.0) 344 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2024-03-07T03:17:28.360 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.Clone (2459 ms) 2024-03-07T03:17:28.361 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.CloneFlatten 2024-03-07T03:17:29.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:28 smithi145 bash[79878]: cluster 2024-03-07T03:17:27.684674+0000 mon.a (mon.0) 345 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2024-03-07T03:17:29.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:28 smithi145 bash[82329]: cluster 2024-03-07T03:17:27.684674+0000 mon.a (mon.0) 345 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2024-03-07T03:17:29.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:28 smithi189 bash[66969]: cluster 2024-03-07T03:17:27.684674+0000 mon.a (mon.0) 345 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2024-03-07T03:17:29.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:29 smithi189 bash[66969]: cluster 2024-03-07T03:17:28.577015+0000 mgr.y (mgr.34107) 100 : cluster 0 pgmap v197: 105 pgs: 105 active+clean; 117 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-03-07T03:17:29.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:29 smithi189 bash[66969]: cluster 2024-03-07T03:17:28.694900+0000 mon.a (mon.0) 346 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2024-03-07T03:17:30.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:29 smithi145 bash[79878]: cluster 2024-03-07T03:17:28.577015+0000 mgr.y (mgr.34107) 100 : cluster 0 pgmap v197: 105 pgs: 105 active+clean; 117 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-03-07T03:17:30.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:29 smithi145 bash[79878]: cluster 2024-03-07T03:17:28.694900+0000 mon.a (mon.0) 346 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2024-03-07T03:17:30.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:29 smithi145 bash[82329]: cluster 2024-03-07T03:17:28.577015+0000 mgr.y (mgr.34107) 100 : cluster 0 pgmap v197: 105 pgs: 105 active+clean; 117 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-03-07T03:17:30.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:29 smithi145 bash[82329]: cluster 2024-03-07T03:17:28.694900+0000 mon.a (mon.0) 346 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2024-03-07T03:17:30.471 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.CloneFlatten (2111 ms) 2024-03-07T03:17:30.471 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Stress 2024-03-07T03:17:30.520 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 8291404~7489946 2024-03-07T03:17:30.552 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 62710953~1000514 2024-03-07T03:17:30.553 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 18504615~5484198 2024-03-07T03:17:30.576 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 17633294~3313014 2024-03-07T03:17:30.578 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 35512231~3417520 2024-03-07T03:17:30.597 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 61096439~2560214 2024-03-07T03:17:30.597 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 9911298~980380 2024-03-07T03:17:30.604 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 35541346~1705699 2024-03-07T03:17:30.606 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap0 2024-03-07T03:17:31.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[82329]: cluster 2024-03-07T03:17:29.702150+0000 mon.a (mon.0) 347 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2024-03-07T03:17:31.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[82329]: audit 2024-03-07T03:17:30.377841+0000 mon.a (mon.0) 348 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:31.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[82329]: audit 2024-03-07T03:17:30.394789+0000 mon.a (mon.0) 349 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:31.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[82329]: audit 2024-03-07T03:17:30.397600+0000 mon.a (mon.0) 350 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:31.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[82329]: audit 2024-03-07T03:17:30.399538+0000 mon.a (mon.0) 351 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:17:31.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[82329]: audit 2024-03-07T03:17:30.410598+0000 mon.a (mon.0) 352 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:31.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[82329]: audit 2024-03-07T03:17:30.487294+0000 mon.a (mon.0) 353 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:17:31.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[82329]: audit 2024-03-07T03:17:30.489177+0000 mon.a (mon.0) 354 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:17:31.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[82329]: audit 2024-03-07T03:17:30.490979+0000 mon.a (mon.0) 355 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:31.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[79878]: cluster 2024-03-07T03:17:29.702150+0000 mon.a (mon.0) 347 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2024-03-07T03:17:31.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[79878]: audit 2024-03-07T03:17:30.377841+0000 mon.a (mon.0) 348 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:31.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[79878]: audit 2024-03-07T03:17:30.394789+0000 mon.a (mon.0) 349 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:31.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[79878]: audit 2024-03-07T03:17:30.397600+0000 mon.a (mon.0) 350 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:31.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[79878]: audit 2024-03-07T03:17:30.399538+0000 mon.a (mon.0) 351 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:17:31.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[79878]: audit 2024-03-07T03:17:30.410598+0000 mon.a (mon.0) 352 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:31.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[79878]: audit 2024-03-07T03:17:30.487294+0000 mon.a (mon.0) 353 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:17:31.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[79878]: audit 2024-03-07T03:17:30.489177+0000 mon.a (mon.0) 354 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:17:31.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:30 smithi145 bash[79878]: audit 2024-03-07T03:17:30.490979+0000 mon.a (mon.0) 355 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:31.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:30 smithi189 bash[66969]: cluster 2024-03-07T03:17:29.702150+0000 mon.a (mon.0) 347 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2024-03-07T03:17:31.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:30 smithi189 bash[66969]: audit 2024-03-07T03:17:30.377841+0000 mon.a (mon.0) 348 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:31.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:30 smithi189 bash[66969]: audit 2024-03-07T03:17:30.394789+0000 mon.a (mon.0) 349 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:31.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:30 smithi189 bash[66969]: audit 2024-03-07T03:17:30.397600+0000 mon.a (mon.0) 350 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:31.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:30 smithi189 bash[66969]: audit 2024-03-07T03:17:30.399538+0000 mon.a (mon.0) 351 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:17:31.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:30 smithi189 bash[66969]: audit 2024-03-07T03:17:30.410598+0000 mon.a (mon.0) 352 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:31.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:30 smithi189 bash[66969]: audit 2024-03-07T03:17:30.487294+0000 mon.a (mon.0) 353 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:17:31.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:30 smithi189 bash[66969]: audit 2024-03-07T03:17:30.489177+0000 mon.a (mon.0) 354 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:17:31.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:30 smithi189 bash[66969]: audit 2024-03-07T03:17:30.490979+0000 mon.a (mon.0) 355 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:clone image236 -> image237 2024-03-07T03:17:31.549 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 135951584 2024-03-07T03:17:31.565 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 109509856~6580256 2024-03-07T03:17:31.604 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 127195645~3667640 2024-03-07T03:17:31.605 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 43788239~1232555 2024-03-07T03:17:31.617 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 40193091~932428 2024-03-07T03:17:31.632 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 134176729~1746383 2024-03-07T03:17:31.647 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 97173778~5359803 2024-03-07T03:17:31.647 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 64874695~1355539 2024-03-07T03:17:31.660 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 8533276~6180276 2024-03-07T03:17:31.681 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap1 2024-03-07T03:17:32.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:31 smithi145 bash[79878]: cephadm 2024-03-07T03:17:30.486706+0000 mgr.y (mgr.34107) 101 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2024-03-07T03:17:32.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:31 smithi145 bash[79878]: cephadm 2024-03-07T03:17:30.492973+0000 mgr.y (mgr.34107) 102 : cephadm 1 Reconfiguring daemon mon.a on smithi145 2024-03-07T03:17:32.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:31 smithi145 bash[79878]: cluster 2024-03-07T03:17:30.577653+0000 mgr.y (mgr.34107) 103 : cluster 0 pgmap v200: 105 pgs: 105 active+clean; 117 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-03-07T03:17:32.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:31 smithi145 bash[79878]: cluster 2024-03-07T03:17:31.416528+0000 mon.a (mon.0) 356 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2024-03-07T03:17:32.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:31 smithi145 bash[82329]: cephadm 2024-03-07T03:17:30.486706+0000 mgr.y (mgr.34107) 101 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2024-03-07T03:17:32.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:31 smithi145 bash[82329]: cephadm 2024-03-07T03:17:30.492973+0000 mgr.y (mgr.34107) 102 : cephadm 1 Reconfiguring daemon mon.a on smithi145 2024-03-07T03:17:32.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:31 smithi145 bash[82329]: cluster 2024-03-07T03:17:30.577653+0000 mgr.y (mgr.34107) 103 : cluster 0 pgmap v200: 105 pgs: 105 active+clean; 117 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-03-07T03:17:32.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:31 smithi145 bash[82329]: cluster 2024-03-07T03:17:31.416528+0000 mon.a (mon.0) 356 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2024-03-07T03:17:32.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:31 smithi189 bash[66969]: cephadm 2024-03-07T03:17:30.486706+0000 mgr.y (mgr.34107) 101 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2024-03-07T03:17:32.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:31 smithi189 bash[66969]: cephadm 2024-03-07T03:17:30.492973+0000 mgr.y (mgr.34107) 102 : cephadm 1 Reconfiguring daemon mon.a on smithi145 2024-03-07T03:17:32.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:31 smithi189 bash[66969]: cluster 2024-03-07T03:17:30.577653+0000 mgr.y (mgr.34107) 103 : cluster 0 pgmap v200: 105 pgs: 105 active+clean; 117 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-03-07T03:17:32.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:31 smithi189 bash[66969]: cluster 2024-03-07T03:17:31.416528+0000 mon.a (mon.0) 356 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2024-03-07T03:17:32.454 INFO:tasks.workunit.client.0.smithi145.stdout:clone image237 -> image238 2024-03-07T03:17:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:32 smithi189 bash[59814]: ts=2024-03-07T03:17:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:32.583 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 2372252~270643 2024-03-07T03:17:32.595 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 72243674~2936588 2024-03-07T03:17:32.603 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 58550787~5774482 2024-03-07T03:17:32.638 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 109413652~3208822 2024-03-07T03:17:32.660 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 125823288~7018470 2024-03-07T03:17:32.695 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 82074177~159080 2024-03-07T03:17:32.703 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 63164392~6929592 2024-03-07T03:17:32.735 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 19877680~6984833 2024-03-07T03:17:32.744 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap2 2024-03-07T03:17:33.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:17:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:17:32] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.43.0" 2024-03-07T03:17:33.465 INFO:tasks.workunit.client.0.smithi145.stdout:clone image238 -> image239 2024-03-07T03:17:33.577 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 175293695 2024-03-07T03:17:33.587 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 7029074~4678737 2024-03-07T03:17:33.613 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 107596234~1159982 2024-03-07T03:17:33.619 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 138238375~2266123 2024-03-07T03:17:33.643 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 87361559~8288477 2024-03-07T03:17:33.653 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 115165083~4131413 2024-03-07T03:17:33.687 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 173795391~1498304 2024-03-07T03:17:33.689 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 114913671~5516749 2024-03-07T03:17:33.707 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 74269104~6587069 2024-03-07T03:17:33.718 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap3 2024-03-07T03:17:33.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[79878]: cluster 2024-03-07T03:17:32.423577+0000 mon.a (mon.0) 357 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2024-03-07T03:17:33.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[79878]: cluster 2024-03-07T03:17:32.578096+0000 mgr.y (mgr.34107) 104 : cluster 0 pgmap v203: 105 pgs: 105 active+clean; 117 MiB data, 697 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:33.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[79878]: audit 2024-03-07T03:17:32.632479+0000 mon.a (mon.0) 358 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:33.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[79878]: audit 2024-03-07T03:17:33.026750+0000 mon.a (mon.0) 359 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:33.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[79878]: audit 2024-03-07T03:17:33.033773+0000 mon.a (mon.0) 360 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:33.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[79878]: audit 2024-03-07T03:17:33.077785+0000 mon.a (mon.0) 361 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:17:33.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[79878]: audit 2024-03-07T03:17:33.079737+0000 mon.a (mon.0) 362 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:17:33.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[79878]: audit 2024-03-07T03:17:33.081675+0000 mon.a (mon.0) 363 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:33.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[82329]: cluster 2024-03-07T03:17:32.423577+0000 mon.a (mon.0) 357 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2024-03-07T03:17:33.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[82329]: cluster 2024-03-07T03:17:32.578096+0000 mgr.y (mgr.34107) 104 : cluster 0 pgmap v203: 105 pgs: 105 active+clean; 117 MiB data, 697 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:33.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[82329]: audit 2024-03-07T03:17:32.632479+0000 mon.a (mon.0) 358 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:33.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[82329]: audit 2024-03-07T03:17:33.026750+0000 mon.a (mon.0) 359 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:33.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[82329]: audit 2024-03-07T03:17:33.033773+0000 mon.a (mon.0) 360 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:33.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[82329]: audit 2024-03-07T03:17:33.077785+0000 mon.a (mon.0) 361 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:17:33.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[82329]: audit 2024-03-07T03:17:33.079737+0000 mon.a (mon.0) 362 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:17:33.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:33 smithi145 bash[82329]: audit 2024-03-07T03:17:33.081675+0000 mon.a (mon.0) 363 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:33.829 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:33 smithi189 bash[59814]: ts=2024-03-07T03:17:33.510Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:33 smithi189 bash[66969]: cluster 2024-03-07T03:17:32.423577+0000 mon.a (mon.0) 357 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2024-03-07T03:17:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:33 smithi189 bash[66969]: cluster 2024-03-07T03:17:32.578096+0000 mgr.y (mgr.34107) 104 : cluster 0 pgmap v203: 105 pgs: 105 active+clean; 117 MiB data, 697 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:17:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:33 smithi189 bash[66969]: audit 2024-03-07T03:17:32.632479+0000 mon.a (mon.0) 358 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:33 smithi189 bash[66969]: audit 2024-03-07T03:17:33.026750+0000 mon.a (mon.0) 359 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:33 smithi189 bash[66969]: audit 2024-03-07T03:17:33.033773+0000 mon.a (mon.0) 360 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:33 smithi189 bash[66969]: audit 2024-03-07T03:17:33.077785+0000 mon.a (mon.0) 361 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:17:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:33 smithi189 bash[66969]: audit 2024-03-07T03:17:33.079737+0000 mon.a (mon.0) 362 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:17:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:33 smithi189 bash[66969]: audit 2024-03-07T03:17:33.081675+0000 mon.a (mon.0) 363 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:34.476 INFO:tasks.workunit.client.0.smithi145.stdout:clone image239 -> image240 2024-03-07T03:17:34.730 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:34 smithi145 bash[79878]: cephadm 2024-03-07T03:17:33.077042+0000 mgr.y (mgr.34107) 105 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2024-03-07T03:17:34.730 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:34 smithi145 bash[79878]: cephadm 2024-03-07T03:17:33.083606+0000 mgr.y (mgr.34107) 106 : cephadm 1 Reconfiguring daemon mgr.y on smithi145 2024-03-07T03:17:34.730 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:34 smithi145 bash[79878]: cluster 2024-03-07T03:17:33.432516+0000 mon.a (mon.0) 364 : cluster 0 osdmap e408: 8 total, 8 up, 8 in 2024-03-07T03:17:34.730 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:34 smithi145 bash[82329]: cephadm 2024-03-07T03:17:33.077042+0000 mgr.y (mgr.34107) 105 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2024-03-07T03:17:34.730 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:34 smithi145 bash[82329]: cephadm 2024-03-07T03:17:33.083606+0000 mgr.y (mgr.34107) 106 : cephadm 1 Reconfiguring daemon mgr.y on smithi145 2024-03-07T03:17:34.730 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:34 smithi145 bash[82329]: cluster 2024-03-07T03:17:33.432516+0000 mon.a (mon.0) 364 : cluster 0 osdmap e408: 8 total, 8 up, 8 in 2024-03-07T03:17:34.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:34 smithi189 bash[66969]: cephadm 2024-03-07T03:17:33.077042+0000 mgr.y (mgr.34107) 105 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2024-03-07T03:17:34.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:34 smithi189 bash[66969]: cephadm 2024-03-07T03:17:33.083606+0000 mgr.y (mgr.34107) 106 : cephadm 1 Reconfiguring daemon mgr.y on smithi145 2024-03-07T03:17:34.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:34 smithi189 bash[66969]: cluster 2024-03-07T03:17:33.432516+0000 mon.a (mon.0) 364 : cluster 0 osdmap e408: 8 total, 8 up, 8 in 2024-03-07T03:17:35.087 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.Stress (4616 ms) 2024-03-07T03:17:35.087 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.NoSnaps_LargerDstObjSize 2024-03-07T03:17:35.578 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.NoSnaps_LargerDstObjSize (491 ms) 2024-03-07T03:17:35.578 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Snaps_LargerDstObjSize 2024-03-07T03:17:35.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:35 smithi145 bash[79878]: cluster 2024-03-07T03:17:34.446929+0000 mon.a (mon.0) 365 : cluster 0 osdmap e409: 8 total, 8 up, 8 in 2024-03-07T03:17:35.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:35 smithi145 bash[79878]: cluster 2024-03-07T03:17:34.578518+0000 mgr.y (mgr.34107) 107 : cluster 0 pgmap v206: 105 pgs: 105 active+clean; 291 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 36 MiB/s wr, 510 op/s 2024-03-07T03:17:35.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:35 smithi145 bash[82329]: cluster 2024-03-07T03:17:34.446929+0000 mon.a (mon.0) 365 : cluster 0 osdmap e409: 8 total, 8 up, 8 in 2024-03-07T03:17:35.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:35 smithi145 bash[82329]: cluster 2024-03-07T03:17:34.578518+0000 mgr.y (mgr.34107) 107 : cluster 0 pgmap v206: 105 pgs: 105 active+clean; 291 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 36 MiB/s wr, 510 op/s 2024-03-07T03:17:35.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:35 smithi189 bash[66969]: cluster 2024-03-07T03:17:34.446929+0000 mon.a (mon.0) 365 : cluster 0 osdmap e409: 8 total, 8 up, 8 in 2024-03-07T03:17:35.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:35 smithi189 bash[66969]: cluster 2024-03-07T03:17:34.578518+0000 mgr.y (mgr.34107) 107 : cluster 0 pgmap v206: 105 pgs: 105 active+clean; 291 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 36 MiB/s wr, 510 op/s 2024-03-07T03:17:37.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[82329]: audit 2024-03-07T03:17:35.605094+0000 mon.a (mon.0) 366 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:37.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[82329]: audit 2024-03-07T03:17:35.614091+0000 mon.a (mon.0) 367 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:37.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[82329]: cephadm 2024-03-07T03:17:35.653884+0000 mgr.y (mgr.34107) 108 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-03-07T03:17:37.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[82329]: audit 2024-03-07T03:17:35.654564+0000 mon.a (mon.0) 368 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:17:37.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[82329]: audit 2024-03-07T03:17:35.656221+0000 mon.a (mon.0) 369 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:17:37.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[82329]: audit 2024-03-07T03:17:35.657799+0000 mon.a (mon.0) 370 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:37.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[82329]: cephadm 2024-03-07T03:17:35.660326+0000 mgr.y (mgr.34107) 109 : cephadm 1 Reconfiguring daemon mon.c on smithi145 2024-03-07T03:17:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[79878]: audit 2024-03-07T03:17:35.605094+0000 mon.a (mon.0) 366 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[79878]: audit 2024-03-07T03:17:35.614091+0000 mon.a (mon.0) 367 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[79878]: cephadm 2024-03-07T03:17:35.653884+0000 mgr.y (mgr.34107) 108 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-03-07T03:17:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[79878]: audit 2024-03-07T03:17:35.654564+0000 mon.a (mon.0) 368 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:17:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[79878]: audit 2024-03-07T03:17:35.656221+0000 mon.a (mon.0) 369 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:17:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[79878]: audit 2024-03-07T03:17:35.657799+0000 mon.a (mon.0) 370 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:36 smithi145 bash[79878]: cephadm 2024-03-07T03:17:35.660326+0000 mgr.y (mgr.34107) 109 : cephadm 1 Reconfiguring daemon mon.c on smithi145 2024-03-07T03:17:37.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:36 smithi189 bash[66969]: audit 2024-03-07T03:17:35.605094+0000 mon.a (mon.0) 366 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:37.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:36 smithi189 bash[66969]: audit 2024-03-07T03:17:35.614091+0000 mon.a (mon.0) 367 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:37.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:36 smithi189 bash[66969]: cephadm 2024-03-07T03:17:35.653884+0000 mgr.y (mgr.34107) 108 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-03-07T03:17:37.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:36 smithi189 bash[66969]: audit 2024-03-07T03:17:35.654564+0000 mon.a (mon.0) 368 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:17:37.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:36 smithi189 bash[66969]: audit 2024-03-07T03:17:35.656221+0000 mon.a (mon.0) 369 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:17:37.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:36 smithi189 bash[66969]: audit 2024-03-07T03:17:35.657799+0000 mon.a (mon.0) 370 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:37.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:36 smithi189 bash[66969]: cephadm 2024-03-07T03:17:35.660326+0000 mgr.y (mgr.34107) 109 : cephadm 1 Reconfiguring daemon mon.c on smithi145 2024-03-07T03:17:37.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:37 smithi145 bash[79878]: cluster 2024-03-07T03:17:36.579236+0000 mgr.y (mgr.34107) 110 : cluster 0 pgmap v207: 105 pgs: 105 active+clean; 291 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 28 MiB/s wr, 395 op/s 2024-03-07T03:17:37.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:37 smithi145 bash[79878]: cluster 2024-03-07T03:17:36.627342+0000 mon.a (mon.0) 371 : cluster 0 osdmap e410: 8 total, 8 up, 8 in 2024-03-07T03:17:37.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:37 smithi145 bash[82329]: cluster 2024-03-07T03:17:36.579236+0000 mgr.y (mgr.34107) 110 : cluster 0 pgmap v207: 105 pgs: 105 active+clean; 291 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 28 MiB/s wr, 395 op/s 2024-03-07T03:17:37.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:37 smithi145 bash[82329]: cluster 2024-03-07T03:17:36.627342+0000 mon.a (mon.0) 371 : cluster 0 osdmap e410: 8 total, 8 up, 8 in 2024-03-07T03:17:38.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:37 smithi189 bash[66969]: cluster 2024-03-07T03:17:36.579236+0000 mgr.y (mgr.34107) 110 : cluster 0 pgmap v207: 105 pgs: 105 active+clean; 291 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 28 MiB/s wr, 395 op/s 2024-03-07T03:17:38.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:37 smithi189 bash[66969]: cluster 2024-03-07T03:17:36.627342+0000 mon.a (mon.0) 371 : cluster 0 osdmap e410: 8 total, 8 up, 8 in 2024-03-07T03:17:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:38 smithi145 bash[79878]: cluster 2024-03-07T03:17:37.629905+0000 mon.a (mon.0) 372 : cluster 0 osdmap e411: 8 total, 8 up, 8 in 2024-03-07T03:17:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:38 smithi145 bash[79878]: audit 2024-03-07T03:17:38.088329+0000 mon.a (mon.0) 373 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:38 smithi145 bash[79878]: audit 2024-03-07T03:17:38.099791+0000 mon.a (mon.0) 374 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:38 smithi145 bash[79878]: audit 2024-03-07T03:17:38.132847+0000 mon.a (mon.0) 375 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:17:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:38 smithi145 bash[79878]: audit 2024-03-07T03:17:38.134201+0000 mon.a (mon.0) 376 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:39.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:38 smithi145 bash[82329]: cluster 2024-03-07T03:17:37.629905+0000 mon.a (mon.0) 372 : cluster 0 osdmap e411: 8 total, 8 up, 8 in 2024-03-07T03:17:39.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:38 smithi145 bash[82329]: audit 2024-03-07T03:17:38.088329+0000 mon.a (mon.0) 373 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:39.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:38 smithi145 bash[82329]: audit 2024-03-07T03:17:38.099791+0000 mon.a (mon.0) 374 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:39.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:38 smithi145 bash[82329]: audit 2024-03-07T03:17:38.132847+0000 mon.a (mon.0) 375 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:17:39.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:38 smithi145 bash[82329]: audit 2024-03-07T03:17:38.134201+0000 mon.a (mon.0) 376 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:39.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:38 smithi189 bash[66969]: cluster 2024-03-07T03:17:37.629905+0000 mon.a (mon.0) 372 : cluster 0 osdmap e411: 8 total, 8 up, 8 in 2024-03-07T03:17:39.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:38 smithi189 bash[66969]: audit 2024-03-07T03:17:38.088329+0000 mon.a (mon.0) 373 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:39.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:38 smithi189 bash[66969]: audit 2024-03-07T03:17:38.099791+0000 mon.a (mon.0) 374 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:39.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:38 smithi189 bash[66969]: audit 2024-03-07T03:17:38.132847+0000 mon.a (mon.0) 375 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:17:39.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:38 smithi189 bash[66969]: audit 2024-03-07T03:17:38.134201+0000 mon.a (mon.0) 376 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:39.902 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:39 smithi145 bash[82329]: cephadm 2024-03-07T03:17:38.132317+0000 mgr.y (mgr.34107) 111 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2024-03-07T03:17:39.902 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:39 smithi145 bash[82329]: cephadm 2024-03-07T03:17:38.139568+0000 mgr.y (mgr.34107) 112 : cephadm 1 Reconfiguring daemon osd.0 on smithi145 2024-03-07T03:17:39.902 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:39 smithi145 bash[82329]: cluster 2024-03-07T03:17:38.580011+0000 mgr.y (mgr.34107) 113 : cluster 0 pgmap v210: 105 pgs: 105 active+clean; 365 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53 MiB/s rd, 41 MiB/s wr, 774 op/s 2024-03-07T03:17:39.902 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:39 smithi145 bash[82329]: cluster 2024-03-07T03:17:38.651360+0000 mon.a (mon.0) 377 : cluster 0 osdmap e412: 8 total, 8 up, 8 in 2024-03-07T03:17:39.902 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:39 smithi145 bash[79878]: cephadm 2024-03-07T03:17:38.132317+0000 mgr.y (mgr.34107) 111 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2024-03-07T03:17:39.902 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:39 smithi145 bash[79878]: cephadm 2024-03-07T03:17:38.139568+0000 mgr.y (mgr.34107) 112 : cephadm 1 Reconfiguring daemon osd.0 on smithi145 2024-03-07T03:17:39.903 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:39 smithi145 bash[79878]: cluster 2024-03-07T03:17:38.580011+0000 mgr.y (mgr.34107) 113 : cluster 0 pgmap v210: 105 pgs: 105 active+clean; 365 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53 MiB/s rd, 41 MiB/s wr, 774 op/s 2024-03-07T03:17:39.903 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:39 smithi145 bash[79878]: cluster 2024-03-07T03:17:38.651360+0000 mon.a (mon.0) 377 : cluster 0 osdmap e412: 8 total, 8 up, 8 in 2024-03-07T03:17:40.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:39 smithi189 bash[66969]: cephadm 2024-03-07T03:17:38.132317+0000 mgr.y (mgr.34107) 111 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2024-03-07T03:17:40.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:39 smithi189 bash[66969]: cephadm 2024-03-07T03:17:38.139568+0000 mgr.y (mgr.34107) 112 : cephadm 1 Reconfiguring daemon osd.0 on smithi145 2024-03-07T03:17:40.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:39 smithi189 bash[66969]: cluster 2024-03-07T03:17:38.580011+0000 mgr.y (mgr.34107) 113 : cluster 0 pgmap v210: 105 pgs: 105 active+clean; 365 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53 MiB/s rd, 41 MiB/s wr, 774 op/s 2024-03-07T03:17:40.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:39 smithi189 bash[66969]: cluster 2024-03-07T03:17:38.651360+0000 mon.a (mon.0) 377 : cluster 0 osdmap e412: 8 total, 8 up, 8 in 2024-03-07T03:17:40.086 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.Snaps_LargerDstObjSize (4507 ms) 2024-03-07T03:17:40.086 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Clone_LargerDstObjSize 2024-03-07T03:17:41.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:40 smithi145 bash[79878]: cluster 2024-03-07T03:17:39.660445+0000 mon.a (mon.0) 378 : cluster 0 osdmap e413: 8 total, 8 up, 8 in 2024-03-07T03:17:41.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:40 smithi145 bash[82329]: cluster 2024-03-07T03:17:39.660445+0000 mon.a (mon.0) 378 : cluster 0 osdmap e413: 8 total, 8 up, 8 in 2024-03-07T03:17:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:40 smithi189 bash[66969]: cluster 2024-03-07T03:17:39.660445+0000 mon.a (mon.0) 378 : cluster 0 osdmap e413: 8 total, 8 up, 8 in 2024-03-07T03:17:42.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:41 smithi145 bash[79878]: cluster 2024-03-07T03:17:40.580654+0000 mgr.y (mgr.34107) 114 : cluster 0 pgmap v213: 105 pgs: 105 active+clean; 365 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 17 MiB/s wr, 486 op/s 2024-03-07T03:17:42.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:41 smithi145 bash[79878]: cluster 2024-03-07T03:17:40.671942+0000 mon.a (mon.0) 379 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2024-03-07T03:17:42.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:41 smithi145 bash[79878]: audit 2024-03-07T03:17:40.884211+0000 mon.a (mon.0) 380 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:42.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:41 smithi145 bash[79878]: audit 2024-03-07T03:17:40.890182+0000 mon.a (mon.0) 381 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:42.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:41 smithi145 bash[79878]: audit 2024-03-07T03:17:40.907915+0000 mon.a (mon.0) 382 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:17:42.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:41 smithi145 bash[79878]: audit 2024-03-07T03:17:40.909033+0000 mon.a (mon.0) 383 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:42.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:41 smithi145 bash[82329]: cluster 2024-03-07T03:17:40.580654+0000 mgr.y (mgr.34107) 114 : cluster 0 pgmap v213: 105 pgs: 105 active+clean; 365 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 17 MiB/s wr, 486 op/s 2024-03-07T03:17:42.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:41 smithi145 bash[82329]: cluster 2024-03-07T03:17:40.671942+0000 mon.a (mon.0) 379 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2024-03-07T03:17:42.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:41 smithi145 bash[82329]: audit 2024-03-07T03:17:40.884211+0000 mon.a (mon.0) 380 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:42.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:41 smithi145 bash[82329]: audit 2024-03-07T03:17:40.890182+0000 mon.a (mon.0) 381 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:42.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:41 smithi145 bash[82329]: audit 2024-03-07T03:17:40.907915+0000 mon.a (mon.0) 382 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:17:42.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:41 smithi145 bash[82329]: audit 2024-03-07T03:17:40.909033+0000 mon.a (mon.0) 383 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:42.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:41 smithi189 bash[66969]: cluster 2024-03-07T03:17:40.580654+0000 mgr.y (mgr.34107) 114 : cluster 0 pgmap v213: 105 pgs: 105 active+clean; 365 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 17 MiB/s wr, 486 op/s 2024-03-07T03:17:42.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:41 smithi189 bash[66969]: cluster 2024-03-07T03:17:40.671942+0000 mon.a (mon.0) 379 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2024-03-07T03:17:42.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:41 smithi189 bash[66969]: audit 2024-03-07T03:17:40.884211+0000 mon.a (mon.0) 380 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:42.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:41 smithi189 bash[66969]: audit 2024-03-07T03:17:40.890182+0000 mon.a (mon.0) 381 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:42.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:41 smithi189 bash[66969]: audit 2024-03-07T03:17:40.907915+0000 mon.a (mon.0) 382 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:17:42.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:41 smithi189 bash[66969]: audit 2024-03-07T03:17:40.909033+0000 mon.a (mon.0) 383 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:42.393 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.Clone_LargerDstObjSize (2308 ms) 2024-03-07T03:17:42.393 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.CloneFlatten_LargerDstObjSize 2024-03-07T03:17:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:42 smithi189 bash[59814]: ts=2024-03-07T03:17:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:43.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:17:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:17:42] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.43.0" 2024-03-07T03:17:43.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:42 smithi145 bash[82329]: cephadm 2024-03-07T03:17:40.907537+0000 mgr.y (mgr.34107) 115 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2024-03-07T03:17:43.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:42 smithi145 bash[82329]: cephadm 2024-03-07T03:17:40.911421+0000 mgr.y (mgr.34107) 116 : cephadm 1 Reconfiguring daemon osd.1 on smithi145 2024-03-07T03:17:43.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:42 smithi145 bash[82329]: cluster 2024-03-07T03:17:41.675116+0000 mon.a (mon.0) 384 : cluster 0 osdmap e415: 8 total, 8 up, 8 in 2024-03-07T03:17:43.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:42 smithi145 bash[79878]: cephadm 2024-03-07T03:17:40.907537+0000 mgr.y (mgr.34107) 115 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2024-03-07T03:17:43.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:42 smithi145 bash[79878]: cephadm 2024-03-07T03:17:40.911421+0000 mgr.y (mgr.34107) 116 : cephadm 1 Reconfiguring daemon osd.1 on smithi145 2024-03-07T03:17:43.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:42 smithi145 bash[79878]: cluster 2024-03-07T03:17:41.675116+0000 mon.a (mon.0) 384 : cluster 0 osdmap e415: 8 total, 8 up, 8 in 2024-03-07T03:17:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:42 smithi189 bash[66969]: cephadm 2024-03-07T03:17:40.907537+0000 mgr.y (mgr.34107) 115 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2024-03-07T03:17:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:42 smithi189 bash[66969]: cephadm 2024-03-07T03:17:40.911421+0000 mgr.y (mgr.34107) 116 : cephadm 1 Reconfiguring daemon osd.1 on smithi145 2024-03-07T03:17:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:42 smithi189 bash[66969]: cluster 2024-03-07T03:17:41.675116+0000 mon.a (mon.0) 384 : cluster 0 osdmap e415: 8 total, 8 up, 8 in 2024-03-07T03:17:43.829 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:43 smithi189 bash[59814]: ts=2024-03-07T03:17:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:43.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:43 smithi189 bash[66969]: cluster 2024-03-07T03:17:42.581111+0000 mgr.y (mgr.34107) 117 : cluster 0 pgmap v216: 105 pgs: 105 active+clean; 365 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:17:43.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:43 smithi189 bash[66969]: cluster 2024-03-07T03:17:43.290861+0000 mon.a (mon.0) 385 : cluster 0 osdmap e416: 8 total, 8 up, 8 in 2024-03-07T03:17:43.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:43 smithi189 bash[66969]: audit 2024-03-07T03:17:43.431914+0000 mon.a (mon.0) 386 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:43.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:43 smithi189 bash[66969]: audit 2024-03-07T03:17:43.439829+0000 mon.a (mon.0) 387 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:43.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:43 smithi189 bash[66969]: audit 2024-03-07T03:17:43.480846+0000 mon.a (mon.0) 388 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:17:43.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:43 smithi189 bash[66969]: audit 2024-03-07T03:17:43.481939+0000 mon.a (mon.0) 389 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:44.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:43 smithi145 bash[79878]: cluster 2024-03-07T03:17:42.581111+0000 mgr.y (mgr.34107) 117 : cluster 0 pgmap v216: 105 pgs: 105 active+clean; 365 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:17:44.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:43 smithi145 bash[79878]: cluster 2024-03-07T03:17:43.290861+0000 mon.a (mon.0) 385 : cluster 0 osdmap e416: 8 total, 8 up, 8 in 2024-03-07T03:17:44.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:43 smithi145 bash[79878]: audit 2024-03-07T03:17:43.431914+0000 mon.a (mon.0) 386 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:44.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:43 smithi145 bash[79878]: audit 2024-03-07T03:17:43.439829+0000 mon.a (mon.0) 387 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:44.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:43 smithi145 bash[79878]: audit 2024-03-07T03:17:43.480846+0000 mon.a (mon.0) 388 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:17:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:43 smithi145 bash[79878]: audit 2024-03-07T03:17:43.481939+0000 mon.a (mon.0) 389 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:43 smithi145 bash[82329]: cluster 2024-03-07T03:17:42.581111+0000 mgr.y (mgr.34107) 117 : cluster 0 pgmap v216: 105 pgs: 105 active+clean; 365 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:17:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:43 smithi145 bash[82329]: cluster 2024-03-07T03:17:43.290861+0000 mon.a (mon.0) 385 : cluster 0 osdmap e416: 8 total, 8 up, 8 in 2024-03-07T03:17:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:43 smithi145 bash[82329]: audit 2024-03-07T03:17:43.431914+0000 mon.a (mon.0) 386 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:43 smithi145 bash[82329]: audit 2024-03-07T03:17:43.439829+0000 mon.a (mon.0) 387 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:43 smithi145 bash[82329]: audit 2024-03-07T03:17:43.480846+0000 mon.a (mon.0) 388 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:17:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:43 smithi145 bash[82329]: audit 2024-03-07T03:17:43.481939+0000 mon.a (mon.0) 389 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:44.954 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:44 smithi145 bash[82329]: cephadm 2024-03-07T03:17:43.480349+0000 mgr.y (mgr.34107) 118 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2024-03-07T03:17:44.954 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:44 smithi145 bash[82329]: cephadm 2024-03-07T03:17:43.485409+0000 mgr.y (mgr.34107) 119 : cephadm 1 Reconfiguring daemon osd.2 on smithi145 2024-03-07T03:17:44.954 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:44 smithi145 bash[82329]: cluster 2024-03-07T03:17:44.447057+0000 mon.a (mon.0) 390 : cluster 0 osdmap e417: 8 total, 8 up, 8 in 2024-03-07T03:17:44.954 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:44 smithi145 bash[79878]: cephadm 2024-03-07T03:17:43.480349+0000 mgr.y (mgr.34107) 118 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2024-03-07T03:17:44.955 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:44 smithi145 bash[79878]: cephadm 2024-03-07T03:17:43.485409+0000 mgr.y (mgr.34107) 119 : cephadm 1 Reconfiguring daemon osd.2 on smithi145 2024-03-07T03:17:44.955 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:44 smithi145 bash[79878]: cluster 2024-03-07T03:17:44.447057+0000 mon.a (mon.0) 390 : cluster 0 osdmap e417: 8 total, 8 up, 8 in 2024-03-07T03:17:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:44 smithi189 bash[66969]: cephadm 2024-03-07T03:17:43.480349+0000 mgr.y (mgr.34107) 118 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2024-03-07T03:17:45.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:44 smithi189 bash[66969]: cephadm 2024-03-07T03:17:43.485409+0000 mgr.y (mgr.34107) 119 : cephadm 1 Reconfiguring daemon osd.2 on smithi145 2024-03-07T03:17:45.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:44 smithi189 bash[66969]: cluster 2024-03-07T03:17:44.447057+0000 mon.a (mon.0) 390 : cluster 0 osdmap e417: 8 total, 8 up, 8 in 2024-03-07T03:17:45.175 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.CloneFlatten_LargerDstObjSize (2781 ms) 2024-03-07T03:17:45.175 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Stress_LargerDstObjSize 2024-03-07T03:17:45.220 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 34720523~7761141 2024-03-07T03:17:45.256 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 65022381~1171208 2024-03-07T03:17:45.258 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 16544649~7905427 2024-03-07T03:17:45.288 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 28428952~122141 2024-03-07T03:17:45.290 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 24701564~7984822 2024-03-07T03:17:45.313 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 21668358~7941866 2024-03-07T03:17:45.318 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 40040628~695722 2024-03-07T03:17:45.323 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 76728815~7157265 2024-03-07T03:17:45.323 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap0 2024-03-07T03:17:45.738 INFO:tasks.workunit.client.0.smithi145.stdout:clone image259 -> image260 2024-03-07T03:17:45.869 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 36797996~2778272 2024-03-07T03:17:45.915 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 73537617~7795684 2024-03-07T03:17:45.923 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 46062689~371705 2024-03-07T03:17:45.931 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 33282720~4627214 2024-03-07T03:17:45.959 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 8138446~6218095 2024-03-07T03:17:45.991 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 15613714~2293693 2024-03-07T03:17:46.025 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 35220658~1890932 2024-03-07T03:17:46.034 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 53393788~2589 2024-03-07T03:17:46.039 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap1 2024-03-07T03:17:46.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:45 smithi145 bash[79878]: cluster 2024-03-07T03:17:44.581753+0000 mgr.y (mgr.34107) 120 : cluster 0 pgmap v219: 105 pgs: 105 active+clean; 445 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 41 MiB/s rd, 20 MiB/s wr, 417 op/s 2024-03-07T03:17:46.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:45 smithi145 bash[82329]: cluster 2024-03-07T03:17:44.581753+0000 mgr.y (mgr.34107) 120 : cluster 0 pgmap v219: 105 pgs: 105 active+clean; 445 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 41 MiB/s rd, 20 MiB/s wr, 417 op/s 2024-03-07T03:17:46.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:45 smithi189 bash[66969]: cluster 2024-03-07T03:17:44.581753+0000 mgr.y (mgr.34107) 120 : cluster 0 pgmap v219: 105 pgs: 105 active+clean; 445 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 41 MiB/s rd, 20 MiB/s wr, 417 op/s 2024-03-07T03:17:46.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:46 smithi145 bash[79878]: cluster 2024-03-07T03:17:45.705040+0000 mon.a (mon.0) 391 : cluster 0 osdmap e418: 8 total, 8 up, 8 in 2024-03-07T03:17:46.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:46 smithi145 bash[79878]: audit 2024-03-07T03:17:46.007358+0000 mon.a (mon.0) 392 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:46.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:46 smithi145 bash[79878]: audit 2024-03-07T03:17:46.013144+0000 mon.a (mon.0) 393 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:46.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:46 smithi145 bash[79878]: audit 2024-03-07T03:17:46.056094+0000 mon.a (mon.0) 394 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:17:46.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:46 smithi145 bash[79878]: audit 2024-03-07T03:17:46.057285+0000 mon.a (mon.0) 395 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:46.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:46 smithi145 bash[82329]: cluster 2024-03-07T03:17:45.705040+0000 mon.a (mon.0) 391 : cluster 0 osdmap e418: 8 total, 8 up, 8 in 2024-03-07T03:17:46.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:46 smithi145 bash[82329]: audit 2024-03-07T03:17:46.007358+0000 mon.a (mon.0) 392 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:46.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:46 smithi145 bash[82329]: audit 2024-03-07T03:17:46.013144+0000 mon.a (mon.0) 393 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:46.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:46 smithi145 bash[82329]: audit 2024-03-07T03:17:46.056094+0000 mon.a (mon.0) 394 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:17:46.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:46 smithi145 bash[82329]: audit 2024-03-07T03:17:46.057285+0000 mon.a (mon.0) 395 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:47.060 INFO:tasks.workunit.client.0.smithi145.stdout:clone image260 -> image261 2024-03-07T03:17:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:46 smithi189 bash[66969]: cluster 2024-03-07T03:17:45.705040+0000 mon.a (mon.0) 391 : cluster 0 osdmap e418: 8 total, 8 up, 8 in 2024-03-07T03:17:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:46 smithi189 bash[66969]: audit 2024-03-07T03:17:46.007358+0000 mon.a (mon.0) 392 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:46 smithi189 bash[66969]: audit 2024-03-07T03:17:46.013144+0000 mon.a (mon.0) 393 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:47.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:46 smithi189 bash[66969]: audit 2024-03-07T03:17:46.056094+0000 mon.a (mon.0) 394 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:17:47.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:46 smithi189 bash[66969]: audit 2024-03-07T03:17:46.057285+0000 mon.a (mon.0) 395 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:47.217 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 24867708~2829574 2024-03-07T03:17:47.263 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 19203004~2782537 2024-03-07T03:17:47.287 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 76259848~7626232 2024-03-07T03:17:47.318 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 25099170~7316931 2024-03-07T03:17:47.331 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 14474196~433383 2024-03-07T03:17:47.354 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 31985273~6968813 2024-03-07T03:17:47.384 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 9747085~3994360 2024-03-07T03:17:47.428 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 11790045~1674405 2024-03-07T03:17:47.431 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap2 2024-03-07T03:17:47.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:47 smithi145 bash[82329]: cephadm 2024-03-07T03:17:46.055500+0000 mgr.y (mgr.34107) 121 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2024-03-07T03:17:47.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:47 smithi145 bash[82329]: cephadm 2024-03-07T03:17:46.060330+0000 mgr.y (mgr.34107) 122 : cephadm 1 Reconfiguring daemon osd.3 on smithi145 2024-03-07T03:17:47.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:47 smithi145 bash[82329]: cluster 2024-03-07T03:17:46.582483+0000 mgr.y (mgr.34107) 123 : cluster 0 pgmap v221: 105 pgs: 105 active+clean; 445 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 34 MiB/s rd, 16 MiB/s wr, 339 op/s 2024-03-07T03:17:47.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:47 smithi145 bash[82329]: cluster 2024-03-07T03:17:47.025217+0000 mon.a (mon.0) 396 : cluster 0 osdmap e419: 8 total, 8 up, 8 in 2024-03-07T03:17:47.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:47 smithi145 bash[82329]: audit 2024-03-07T03:17:47.631848+0000 mon.a (mon.0) 397 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:47.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:47 smithi145 bash[79878]: cephadm 2024-03-07T03:17:46.055500+0000 mgr.y (mgr.34107) 121 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2024-03-07T03:17:47.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:47 smithi145 bash[79878]: cephadm 2024-03-07T03:17:46.060330+0000 mgr.y (mgr.34107) 122 : cephadm 1 Reconfiguring daemon osd.3 on smithi145 2024-03-07T03:17:47.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:47 smithi145 bash[79878]: cluster 2024-03-07T03:17:46.582483+0000 mgr.y (mgr.34107) 123 : cluster 0 pgmap v221: 105 pgs: 105 active+clean; 445 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 34 MiB/s rd, 16 MiB/s wr, 339 op/s 2024-03-07T03:17:47.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:47 smithi145 bash[79878]: cluster 2024-03-07T03:17:47.025217+0000 mon.a (mon.0) 396 : cluster 0 osdmap e419: 8 total, 8 up, 8 in 2024-03-07T03:17:47.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:47 smithi145 bash[79878]: audit 2024-03-07T03:17:47.631848+0000 mon.a (mon.0) 397 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:48.054 INFO:tasks.workunit.client.0.smithi145.stdout:clone image261 -> image262 2024-03-07T03:17:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:47 smithi189 bash[66969]: cephadm 2024-03-07T03:17:46.055500+0000 mgr.y (mgr.34107) 121 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2024-03-07T03:17:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:47 smithi189 bash[66969]: cephadm 2024-03-07T03:17:46.060330+0000 mgr.y (mgr.34107) 122 : cephadm 1 Reconfiguring daemon osd.3 on smithi145 2024-03-07T03:17:48.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:47 smithi189 bash[66969]: cluster 2024-03-07T03:17:46.582483+0000 mgr.y (mgr.34107) 123 : cluster 0 pgmap v221: 105 pgs: 105 active+clean; 445 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 34 MiB/s rd, 16 MiB/s wr, 339 op/s 2024-03-07T03:17:48.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:47 smithi189 bash[66969]: cluster 2024-03-07T03:17:47.025217+0000 mon.a (mon.0) 396 : cluster 0 osdmap e419: 8 total, 8 up, 8 in 2024-03-07T03:17:48.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:47 smithi189 bash[66969]: audit 2024-03-07T03:17:47.631848+0000 mon.a (mon.0) 397 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:48.259 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 54357484~7521086 2024-03-07T03:17:48.304 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 5350573~1774398 2024-03-07T03:17:48.327 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 29233915~3775674 2024-03-07T03:17:48.350 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 55998255~4187072 2024-03-07T03:17:48.354 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 54521309~5734317 2024-03-07T03:17:48.378 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 30479884~340139 2024-03-07T03:17:48.381 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 78667185~3709070 2024-03-07T03:17:48.418 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 65211863~5015944 2024-03-07T03:17:48.427 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap3 2024-03-07T03:17:49.061 INFO:tasks.workunit.client.0.smithi145.stdout:clone image262 -> image263 2024-03-07T03:17:49.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[79878]: cluster 2024-03-07T03:17:48.029271+0000 mon.a (mon.0) 398 : cluster 0 osdmap e420: 8 total, 8 up, 8 in 2024-03-07T03:17:49.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[79878]: cluster 2024-03-07T03:17:48.583445+0000 mgr.y (mgr.34107) 124 : cluster 0 pgmap v224: 105 pgs: 105 active+clean; 645 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 44 MiB/s wr, 479 op/s 2024-03-07T03:17:49.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[79878]: audit 2024-03-07T03:17:48.613735+0000 mon.a (mon.0) 399 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:49.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[79878]: audit 2024-03-07T03:17:48.624180+0000 mon.a (mon.0) 400 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:49.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[79878]: cephadm 2024-03-07T03:17:48.630089+0000 mgr.y (mgr.34107) 125 : cephadm 1 Reconfiguring mds.foo.smithi145.zmgvgw (monmap changed)... 2024-03-07T03:17:49.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[79878]: audit 2024-03-07T03:17:48.630697+0000 mon.a (mon.0) 401 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi145.zmgvgw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:17:49.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[79878]: audit 2024-03-07T03:17:48.632243+0000 mon.a (mon.0) 402 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:49.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[79878]: cephadm 2024-03-07T03:17:48.633903+0000 mgr.y (mgr.34107) 126 : cephadm 1 Reconfiguring daemon mds.foo.smithi145.zmgvgw on smithi145 2024-03-07T03:17:49.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[82329]: cluster 2024-03-07T03:17:48.029271+0000 mon.a (mon.0) 398 : cluster 0 osdmap e420: 8 total, 8 up, 8 in 2024-03-07T03:17:49.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[82329]: cluster 2024-03-07T03:17:48.583445+0000 mgr.y (mgr.34107) 124 : cluster 0 pgmap v224: 105 pgs: 105 active+clean; 645 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 44 MiB/s wr, 479 op/s 2024-03-07T03:17:49.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[82329]: audit 2024-03-07T03:17:48.613735+0000 mon.a (mon.0) 399 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:49.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[82329]: audit 2024-03-07T03:17:48.624180+0000 mon.a (mon.0) 400 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:49.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[82329]: cephadm 2024-03-07T03:17:48.630089+0000 mgr.y (mgr.34107) 125 : cephadm 1 Reconfiguring mds.foo.smithi145.zmgvgw (monmap changed)... 2024-03-07T03:17:49.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[82329]: audit 2024-03-07T03:17:48.630697+0000 mon.a (mon.0) 401 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi145.zmgvgw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:17:49.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[82329]: audit 2024-03-07T03:17:48.632243+0000 mon.a (mon.0) 402 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:49.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:49 smithi145 bash[82329]: cephadm 2024-03-07T03:17:48.633903+0000 mgr.y (mgr.34107) 126 : cephadm 1 Reconfiguring daemon mds.foo.smithi145.zmgvgw on smithi145 2024-03-07T03:17:49.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:49 smithi189 bash[66969]: cluster 2024-03-07T03:17:48.029271+0000 mon.a (mon.0) 398 : cluster 0 osdmap e420: 8 total, 8 up, 8 in 2024-03-07T03:17:49.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:49 smithi189 bash[66969]: cluster 2024-03-07T03:17:48.583445+0000 mgr.y (mgr.34107) 124 : cluster 0 pgmap v224: 105 pgs: 105 active+clean; 645 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 44 MiB/s wr, 479 op/s 2024-03-07T03:17:49.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:49 smithi189 bash[66969]: audit 2024-03-07T03:17:48.613735+0000 mon.a (mon.0) 399 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:49.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:49 smithi189 bash[66969]: audit 2024-03-07T03:17:48.624180+0000 mon.a (mon.0) 400 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:49.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:49 smithi189 bash[66969]: cephadm 2024-03-07T03:17:48.630089+0000 mgr.y (mgr.34107) 125 : cephadm 1 Reconfiguring mds.foo.smithi145.zmgvgw (monmap changed)... 2024-03-07T03:17:49.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:49 smithi189 bash[66969]: audit 2024-03-07T03:17:48.630697+0000 mon.a (mon.0) 401 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi145.zmgvgw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:17:49.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:49 smithi189 bash[66969]: audit 2024-03-07T03:17:48.632243+0000 mon.a (mon.0) 402 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:49.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:49 smithi189 bash[66969]: cephadm 2024-03-07T03:17:48.633903+0000 mgr.y (mgr.34107) 126 : cephadm 1 Reconfiguring daemon mds.foo.smithi145.zmgvgw on smithi145 2024-03-07T03:17:49.776 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.Stress_LargerDstObjSize (4601 ms) 2024-03-07T03:17:49.776 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.NoSnaps_SmallerDstObjSize 2024-03-07T03:17:50.170 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.NoSnaps_SmallerDstObjSize (394 ms) 2024-03-07T03:17:50.170 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Snaps_SmallerDstObjSize 2024-03-07T03:17:50.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:50 smithi145 bash[79878]: cluster 2024-03-07T03:17:49.041987+0000 mon.a (mon.0) 403 : cluster 0 osdmap e421: 8 total, 8 up, 8 in 2024-03-07T03:17:50.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:50 smithi145 bash[82329]: cluster 2024-03-07T03:17:49.041987+0000 mon.a (mon.0) 403 : cluster 0 osdmap e421: 8 total, 8 up, 8 in 2024-03-07T03:17:50.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:50 smithi189 bash[66969]: cluster 2024-03-07T03:17:49.041987+0000 mon.a (mon.0) 403 : cluster 0 osdmap e421: 8 total, 8 up, 8 in 2024-03-07T03:17:51.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:51 smithi145 bash[79878]: cluster 2024-03-07T03:17:50.584095+0000 mgr.y (mgr.34107) 127 : cluster 0 pgmap v226: 105 pgs: 105 active+clean; 645 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 38 MiB/s wr, 406 op/s 2024-03-07T03:17:51.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:51 smithi145 bash[82329]: cluster 2024-03-07T03:17:50.584095+0000 mgr.y (mgr.34107) 127 : cluster 0 pgmap v226: 105 pgs: 105 active+clean; 645 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 38 MiB/s wr, 406 op/s 2024-03-07T03:17:51.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:51 smithi189 bash[66969]: cluster 2024-03-07T03:17:50.584095+0000 mgr.y (mgr.34107) 127 : cluster 0 pgmap v226: 105 pgs: 105 active+clean; 645 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 38 MiB/s wr, 406 op/s 2024-03-07T03:17:52.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:52 smithi189 bash[66969]: cluster 2024-03-07T03:17:51.048022+0000 mon.a (mon.0) 404 : cluster 0 osdmap e422: 8 total, 8 up, 8 in 2024-03-07T03:17:52.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:52 smithi189 bash[66969]: audit 2024-03-07T03:17:51.069736+0000 mon.a (mon.0) 405 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:52.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:52 smithi189 bash[66969]: audit 2024-03-07T03:17:51.077638+0000 mon.a (mon.0) 406 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:52.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:52 smithi189 bash[66969]: cephadm 2024-03-07T03:17:51.103378+0000 mgr.y (mgr.34107) 128 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-03-07T03:17:52.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:52 smithi189 bash[66969]: audit 2024-03-07T03:17:51.104084+0000 mon.a (mon.0) 407 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:17:52.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:52 smithi189 bash[66969]: audit 2024-03-07T03:17:51.105415+0000 mon.a (mon.0) 408 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:17:52.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:52 smithi189 bash[66969]: audit 2024-03-07T03:17:51.106579+0000 mon.a (mon.0) 409 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:52.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:52 smithi189 bash[66969]: cephadm 2024-03-07T03:17:51.107957+0000 mgr.y (mgr.34107) 129 : cephadm 1 Reconfiguring daemon mon.b on smithi189 2024-03-07T03:17:52.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[79878]: cluster 2024-03-07T03:17:51.048022+0000 mon.a (mon.0) 404 : cluster 0 osdmap e422: 8 total, 8 up, 8 in 2024-03-07T03:17:52.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[79878]: audit 2024-03-07T03:17:51.069736+0000 mon.a (mon.0) 405 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:52.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[79878]: audit 2024-03-07T03:17:51.077638+0000 mon.a (mon.0) 406 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:52.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[79878]: cephadm 2024-03-07T03:17:51.103378+0000 mgr.y (mgr.34107) 128 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-03-07T03:17:52.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[79878]: audit 2024-03-07T03:17:51.104084+0000 mon.a (mon.0) 407 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:17:52.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[79878]: audit 2024-03-07T03:17:51.105415+0000 mon.a (mon.0) 408 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:17:52.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[79878]: audit 2024-03-07T03:17:51.106579+0000 mon.a (mon.0) 409 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:52.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[79878]: cephadm 2024-03-07T03:17:51.107957+0000 mgr.y (mgr.34107) 129 : cephadm 1 Reconfiguring daemon mon.b on smithi189 2024-03-07T03:17:52.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[82329]: cluster 2024-03-07T03:17:51.048022+0000 mon.a (mon.0) 404 : cluster 0 osdmap e422: 8 total, 8 up, 8 in 2024-03-07T03:17:52.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[82329]: audit 2024-03-07T03:17:51.069736+0000 mon.a (mon.0) 405 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:52.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[82329]: audit 2024-03-07T03:17:51.077638+0000 mon.a (mon.0) 406 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:52.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[82329]: cephadm 2024-03-07T03:17:51.103378+0000 mgr.y (mgr.34107) 128 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-03-07T03:17:52.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[82329]: audit 2024-03-07T03:17:51.104084+0000 mon.a (mon.0) 407 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:17:52.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[82329]: audit 2024-03-07T03:17:51.105415+0000 mon.a (mon.0) 408 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:17:52.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[82329]: audit 2024-03-07T03:17:51.106579+0000 mon.a (mon.0) 409 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:52.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[82329]: cephadm 2024-03-07T03:17:51.107957+0000 mgr.y (mgr.34107) 129 : cephadm 1 Reconfiguring daemon mon.b on smithi189 2024-03-07T03:17:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:52 smithi189 bash[59814]: ts=2024-03-07T03:17:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:53.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:17:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:17:52] "GET /metrics HTTP/1.1" 200 36333 "" "Prometheus/2.43.0" 2024-03-07T03:17:53.466 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:53 smithi189 bash[66969]: cluster 2024-03-07T03:17:52.083996+0000 mon.a (mon.0) 410 : cluster 0 osdmap e423: 8 total, 8 up, 8 in 2024-03-07T03:17:53.466 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:53 smithi189 bash[66969]: cluster 2024-03-07T03:17:52.584814+0000 mgr.y (mgr.34107) 130 : cluster 0 pgmap v229: 105 pgs: 105 active+clean; 645 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 40 MiB/s wr, 434 op/s 2024-03-07T03:17:53.466 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:53 smithi189 bash[66969]: audit 2024-03-07T03:17:53.008475+0000 mon.a (mon.0) 411 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:53.466 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:53 smithi189 bash[66969]: audit 2024-03-07T03:17:53.021295+0000 mon.a (mon.0) 412 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:53.466 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:53 smithi189 bash[66969]: audit 2024-03-07T03:17:53.027679+0000 mon.a (mon.0) 413 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:17:53.466 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:53 smithi189 bash[66969]: audit 2024-03-07T03:17:53.029500+0000 mon.a (mon.0) 414 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:17:53.466 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:53 smithi189 bash[66969]: audit 2024-03-07T03:17:53.031137+0000 mon.a (mon.0) 415 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:53.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[82329]: cluster 2024-03-07T03:17:52.083996+0000 mon.a (mon.0) 410 : cluster 0 osdmap e423: 8 total, 8 up, 8 in 2024-03-07T03:17:53.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[82329]: cluster 2024-03-07T03:17:52.584814+0000 mgr.y (mgr.34107) 130 : cluster 0 pgmap v229: 105 pgs: 105 active+clean; 645 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 40 MiB/s wr, 434 op/s 2024-03-07T03:17:53.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[82329]: audit 2024-03-07T03:17:53.008475+0000 mon.a (mon.0) 411 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:53.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[82329]: audit 2024-03-07T03:17:53.021295+0000 mon.a (mon.0) 412 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:53.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[82329]: audit 2024-03-07T03:17:53.027679+0000 mon.a (mon.0) 413 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:17:53.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[82329]: audit 2024-03-07T03:17:53.029500+0000 mon.a (mon.0) 414 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:17:53.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[82329]: audit 2024-03-07T03:17:53.031137+0000 mon.a (mon.0) 415 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:53.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[79878]: cluster 2024-03-07T03:17:52.083996+0000 mon.a (mon.0) 410 : cluster 0 osdmap e423: 8 total, 8 up, 8 in 2024-03-07T03:17:53.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[79878]: cluster 2024-03-07T03:17:52.584814+0000 mgr.y (mgr.34107) 130 : cluster 0 pgmap v229: 105 pgs: 105 active+clean; 645 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 40 MiB/s wr, 434 op/s 2024-03-07T03:17:53.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[79878]: audit 2024-03-07T03:17:53.008475+0000 mon.a (mon.0) 411 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:53.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[79878]: audit 2024-03-07T03:17:53.021295+0000 mon.a (mon.0) 412 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:53.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[79878]: audit 2024-03-07T03:17:53.027679+0000 mon.a (mon.0) 413 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:17:53.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[79878]: audit 2024-03-07T03:17:53.029500+0000 mon.a (mon.0) 414 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:17:53.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:53 smithi145 bash[79878]: audit 2024-03-07T03:17:53.031137+0000 mon.a (mon.0) 415 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:53.829 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:17:53 smithi189 bash[59814]: ts=2024-03-07T03:17:53.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:54.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:54 smithi189 bash[66969]: cephadm 2024-03-07T03:17:53.026953+0000 mgr.y (mgr.34107) 131 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2024-03-07T03:17:54.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:54 smithi189 bash[66969]: cephadm 2024-03-07T03:17:53.033058+0000 mgr.y (mgr.34107) 132 : cephadm 1 Reconfiguring daemon mgr.x on smithi189 2024-03-07T03:17:54.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:54 smithi189 bash[66969]: cluster 2024-03-07T03:17:53.098464+0000 mon.a (mon.0) 416 : cluster 0 osdmap e424: 8 total, 8 up, 8 in 2024-03-07T03:17:54.531 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.Snaps_SmallerDstObjSize (4362 ms) 2024-03-07T03:17:54.531 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Clone_SmallerDstObjSize 2024-03-07T03:17:54.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:54 smithi145 bash[79878]: cephadm 2024-03-07T03:17:53.026953+0000 mgr.y (mgr.34107) 131 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2024-03-07T03:17:54.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:54 smithi145 bash[79878]: cephadm 2024-03-07T03:17:53.033058+0000 mgr.y (mgr.34107) 132 : cephadm 1 Reconfiguring daemon mgr.x on smithi189 2024-03-07T03:17:54.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:54 smithi145 bash[79878]: cluster 2024-03-07T03:17:53.098464+0000 mon.a (mon.0) 416 : cluster 0 osdmap e424: 8 total, 8 up, 8 in 2024-03-07T03:17:54.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:54 smithi145 bash[82329]: cephadm 2024-03-07T03:17:53.026953+0000 mgr.y (mgr.34107) 131 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2024-03-07T03:17:54.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:54 smithi145 bash[82329]: cephadm 2024-03-07T03:17:53.033058+0000 mgr.y (mgr.34107) 132 : cephadm 1 Reconfiguring daemon mgr.x on smithi189 2024-03-07T03:17:54.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:54 smithi145 bash[82329]: cluster 2024-03-07T03:17:53.098464+0000 mon.a (mon.0) 416 : cluster 0 osdmap e424: 8 total, 8 up, 8 in 2024-03-07T03:17:55.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:55 smithi145 bash[79878]: cluster 2024-03-07T03:17:54.104280+0000 mon.a (mon.0) 417 : cluster 0 osdmap e425: 8 total, 8 up, 8 in 2024-03-07T03:17:55.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:55 smithi145 bash[79878]: cluster 2024-03-07T03:17:54.585504+0000 mgr.y (mgr.34107) 133 : cluster 0 pgmap v232: 105 pgs: 105 active+clean; 693 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 12 MiB/s wr, 417 op/s 2024-03-07T03:17:55.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:55 smithi145 bash[82329]: cluster 2024-03-07T03:17:54.104280+0000 mon.a (mon.0) 417 : cluster 0 osdmap e425: 8 total, 8 up, 8 in 2024-03-07T03:17:55.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:55 smithi145 bash[82329]: cluster 2024-03-07T03:17:54.585504+0000 mgr.y (mgr.34107) 133 : cluster 0 pgmap v232: 105 pgs: 105 active+clean; 693 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 12 MiB/s wr, 417 op/s 2024-03-07T03:17:55.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:55 smithi189 bash[66969]: cluster 2024-03-07T03:17:54.104280+0000 mon.a (mon.0) 417 : cluster 0 osdmap e425: 8 total, 8 up, 8 in 2024-03-07T03:17:55.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:55 smithi189 bash[66969]: cluster 2024-03-07T03:17:54.585504+0000 mgr.y (mgr.34107) 133 : cluster 0 pgmap v232: 105 pgs: 105 active+clean; 693 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 12 MiB/s wr, 417 op/s 2024-03-07T03:17:56.372 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:56 smithi189 bash[66969]: cluster 2024-03-07T03:17:55.114590+0000 mon.a (mon.0) 418 : cluster 0 osdmap e426: 8 total, 8 up, 8 in 2024-03-07T03:17:56.373 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:56 smithi189 bash[66969]: audit 2024-03-07T03:17:55.154673+0000 mon.a (mon.0) 419 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:56.373 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:56 smithi189 bash[66969]: audit 2024-03-07T03:17:55.169133+0000 mon.a (mon.0) 420 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:56.373 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:56 smithi189 bash[66969]: cephadm 2024-03-07T03:17:55.203213+0000 mgr.y (mgr.34107) 134 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2024-03-07T03:17:56.373 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:56 smithi189 bash[66969]: audit 2024-03-07T03:17:55.204054+0000 mon.a (mon.0) 421 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:17:56.373 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:56 smithi189 bash[66969]: audit 2024-03-07T03:17:55.205974+0000 mon.a (mon.0) 422 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:56.373 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:56 smithi189 bash[66969]: cephadm 2024-03-07T03:17:55.212079+0000 mgr.y (mgr.34107) 135 : cephadm 1 Reconfiguring daemon osd.4 on smithi189 2024-03-07T03:17:56.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[79878]: cluster 2024-03-07T03:17:55.114590+0000 mon.a (mon.0) 418 : cluster 0 osdmap e426: 8 total, 8 up, 8 in 2024-03-07T03:17:56.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[79878]: audit 2024-03-07T03:17:55.154673+0000 mon.a (mon.0) 419 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:56.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[79878]: audit 2024-03-07T03:17:55.169133+0000 mon.a (mon.0) 420 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:56.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[79878]: cephadm 2024-03-07T03:17:55.203213+0000 mgr.y (mgr.34107) 134 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2024-03-07T03:17:56.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[79878]: audit 2024-03-07T03:17:55.204054+0000 mon.a (mon.0) 421 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:17:56.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[79878]: audit 2024-03-07T03:17:55.205974+0000 mon.a (mon.0) 422 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:56.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[79878]: cephadm 2024-03-07T03:17:55.212079+0000 mgr.y (mgr.34107) 135 : cephadm 1 Reconfiguring daemon osd.4 on smithi189 2024-03-07T03:17:56.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[82329]: cluster 2024-03-07T03:17:55.114590+0000 mon.a (mon.0) 418 : cluster 0 osdmap e426: 8 total, 8 up, 8 in 2024-03-07T03:17:56.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[82329]: audit 2024-03-07T03:17:55.154673+0000 mon.a (mon.0) 419 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:56.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[82329]: audit 2024-03-07T03:17:55.169133+0000 mon.a (mon.0) 420 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:56.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[82329]: cephadm 2024-03-07T03:17:55.203213+0000 mgr.y (mgr.34107) 134 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2024-03-07T03:17:56.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[82329]: audit 2024-03-07T03:17:55.204054+0000 mon.a (mon.0) 421 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:17:56.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[82329]: audit 2024-03-07T03:17:55.205974+0000 mon.a (mon.0) 422 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:56.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:56 smithi145 bash[82329]: cephadm 2024-03-07T03:17:55.212079+0000 mgr.y (mgr.34107) 135 : cephadm 1 Reconfiguring daemon osd.4 on smithi189 2024-03-07T03:17:56.806 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.Clone_SmallerDstObjSize (2275 ms) 2024-03-07T03:17:56.806 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.CloneFlatten_SmallerDstObjSize 2024-03-07T03:17:57.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:57 smithi145 bash[79878]: cluster 2024-03-07T03:17:56.171994+0000 mon.a (mon.0) 423 : cluster 0 osdmap e427: 8 total, 8 up, 8 in 2024-03-07T03:17:57.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:57 smithi145 bash[79878]: cluster 2024-03-07T03:17:56.586461+0000 mgr.y (mgr.34107) 136 : cluster 0 pgmap v235: 105 pgs: 105 active+clean; 693 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 12 MiB/s wr, 417 op/s 2024-03-07T03:17:57.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:57 smithi145 bash[82329]: cluster 2024-03-07T03:17:56.171994+0000 mon.a (mon.0) 423 : cluster 0 osdmap e427: 8 total, 8 up, 8 in 2024-03-07T03:17:57.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:57 smithi145 bash[82329]: cluster 2024-03-07T03:17:56.586461+0000 mgr.y (mgr.34107) 136 : cluster 0 pgmap v235: 105 pgs: 105 active+clean; 693 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 12 MiB/s wr, 417 op/s 2024-03-07T03:17:57.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:57 smithi189 bash[66969]: cluster 2024-03-07T03:17:56.171994+0000 mon.a (mon.0) 423 : cluster 0 osdmap e427: 8 total, 8 up, 8 in 2024-03-07T03:17:57.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:57 smithi189 bash[66969]: cluster 2024-03-07T03:17:56.586461+0000 mgr.y (mgr.34107) 136 : cluster 0 pgmap v235: 105 pgs: 105 active+clean; 693 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 12 MiB/s wr, 417 op/s 2024-03-07T03:17:58.487 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:58 smithi189 bash[66969]: cluster 2024-03-07T03:17:57.185191+0000 mon.a (mon.0) 424 : cluster 0 osdmap e428: 8 total, 8 up, 8 in 2024-03-07T03:17:58.488 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:58 smithi189 bash[66969]: audit 2024-03-07T03:17:57.245051+0000 mon.a (mon.0) 425 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:58.488 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:58 smithi189 bash[66969]: audit 2024-03-07T03:17:57.259742+0000 mon.a (mon.0) 426 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:58.488 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:58 smithi189 bash[66969]: cephadm 2024-03-07T03:17:57.281552+0000 mgr.y (mgr.34107) 137 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2024-03-07T03:17:58.488 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:58 smithi189 bash[66969]: audit 2024-03-07T03:17:57.282240+0000 mon.a (mon.0) 427 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-07T03:17:58.488 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:58 smithi189 bash[66969]: audit 2024-03-07T03:17:57.284237+0000 mon.a (mon.0) 428 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:58.488 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:58 smithi189 bash[66969]: cephadm 2024-03-07T03:17:57.289916+0000 mgr.y (mgr.34107) 138 : cephadm 1 Reconfiguring daemon osd.5 on smithi189 2024-03-07T03:17:58.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[79878]: cluster 2024-03-07T03:17:57.185191+0000 mon.a (mon.0) 424 : cluster 0 osdmap e428: 8 total, 8 up, 8 in 2024-03-07T03:17:58.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[79878]: audit 2024-03-07T03:17:57.245051+0000 mon.a (mon.0) 425 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:58.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[79878]: audit 2024-03-07T03:17:57.259742+0000 mon.a (mon.0) 426 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:58.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[79878]: cephadm 2024-03-07T03:17:57.281552+0000 mgr.y (mgr.34107) 137 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2024-03-07T03:17:58.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[79878]: audit 2024-03-07T03:17:57.282240+0000 mon.a (mon.0) 427 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-07T03:17:58.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[79878]: audit 2024-03-07T03:17:57.284237+0000 mon.a (mon.0) 428 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:58.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[79878]: cephadm 2024-03-07T03:17:57.289916+0000 mgr.y (mgr.34107) 138 : cephadm 1 Reconfiguring daemon osd.5 on smithi189 2024-03-07T03:17:58.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[82329]: cluster 2024-03-07T03:17:57.185191+0000 mon.a (mon.0) 424 : cluster 0 osdmap e428: 8 total, 8 up, 8 in 2024-03-07T03:17:58.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[82329]: audit 2024-03-07T03:17:57.245051+0000 mon.a (mon.0) 425 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:58.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[82329]: audit 2024-03-07T03:17:57.259742+0000 mon.a (mon.0) 426 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:17:58.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[82329]: cephadm 2024-03-07T03:17:57.281552+0000 mgr.y (mgr.34107) 137 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2024-03-07T03:17:58.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[82329]: audit 2024-03-07T03:17:57.282240+0000 mon.a (mon.0) 427 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-07T03:17:58.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[82329]: audit 2024-03-07T03:17:57.284237+0000 mon.a (mon.0) 428 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:58.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:58 smithi145 bash[82329]: cephadm 2024-03-07T03:17:57.289916+0000 mgr.y (mgr.34107) 138 : cephadm 1 Reconfiguring daemon osd.5 on smithi189 2024-03-07T03:17:58.897 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.CloneFlatten_SmallerDstObjSize (2091 ms) 2024-03-07T03:17:58.898 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Stress_SmallerDstObjSize 2024-03-07T03:17:58.933 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 18998021~872177 2024-03-07T03:17:58.946 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 24468843~2767120 2024-03-07T03:17:58.947 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 46068638~6215166 2024-03-07T03:17:58.972 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 9616603~977424 2024-03-07T03:17:58.972 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 3023535~3524220 2024-03-07T03:17:58.987 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 83047529~360473 2024-03-07T03:17:58.988 INFO:tasks.workunit.client.0.smithi145.stdout:write: A 56042694~4797933 2024-03-07T03:17:59.003 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 25643241~2937123 2024-03-07T03:17:59.003 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap0 2024-03-07T03:17:59.311 INFO:tasks.workunit.client.0.smithi145.stdout:clone image282 -> image283 2024-03-07T03:17:59.430 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 30864996~6211734 2024-03-07T03:17:59.463 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 19914403~555262 2024-03-07T03:17:59.474 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 70833839~1019634 2024-03-07T03:17:59.483 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 49454281~8153131 2024-03-07T03:17:59.514 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 12406700~931869 2024-03-07T03:17:59.522 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 39010126~7950035 2024-03-07T03:17:59.531 INFO:tasks.workunit.client.0.smithi145.stdout:write: B 433604~8205007 2024-03-07T03:17:59.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:59 smithi145 bash[79878]: cluster 2024-03-07T03:17:58.266072+0000 mon.a (mon.0) 429 : cluster 0 osdmap e429: 8 total, 8 up, 8 in 2024-03-07T03:17:59.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:17:59 smithi145 bash[79878]: cluster 2024-03-07T03:17:58.588327+0000 mgr.y (mgr.34107) 139 : cluster 0 pgmap v238: 105 pgs: 105 active+clean; 773 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 20 MiB/s wr, 538 op/s 2024-03-07T03:17:59.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:59 smithi145 bash[82329]: cluster 2024-03-07T03:17:58.266072+0000 mon.a (mon.0) 429 : cluster 0 osdmap e429: 8 total, 8 up, 8 in 2024-03-07T03:17:59.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:17:59 smithi145 bash[82329]: cluster 2024-03-07T03:17:58.588327+0000 mgr.y (mgr.34107) 139 : cluster 0 pgmap v238: 105 pgs: 105 active+clean; 773 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 20 MiB/s wr, 538 op/s 2024-03-07T03:17:59.564 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 10972127~7676500 2024-03-07T03:17:59.566 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap1 2024-03-07T03:17:59.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:59 smithi189 bash[66969]: cluster 2024-03-07T03:17:58.266072+0000 mon.a (mon.0) 429 : cluster 0 osdmap e429: 8 total, 8 up, 8 in 2024-03-07T03:17:59.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:17:59 smithi189 bash[66969]: cluster 2024-03-07T03:17:58.588327+0000 mgr.y (mgr.34107) 139 : cluster 0 pgmap v238: 105 pgs: 105 active+clean; 773 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 20 MiB/s wr, 538 op/s 2024-03-07T03:18:00.489 INFO:tasks.workunit.client.0.smithi145.stdout:clone image283 -> image284 2024-03-07T03:18:00.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[79878]: cluster 2024-03-07T03:17:59.284358+0000 mon.a (mon.0) 430 : cluster 0 osdmap e430: 8 total, 8 up, 8 in 2024-03-07T03:18:00.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[79878]: audit 2024-03-07T03:17:59.440097+0000 mon.a (mon.0) 431 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:00.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[79878]: audit 2024-03-07T03:17:59.453665+0000 mon.a (mon.0) 432 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:00.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[79878]: cephadm 2024-03-07T03:17:59.458010+0000 mgr.y (mgr.34107) 140 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2024-03-07T03:18:00.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[79878]: audit 2024-03-07T03:17:59.458715+0000 mon.a (mon.0) 433 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-07T03:18:00.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[79878]: audit 2024-03-07T03:17:59.460888+0000 mon.a (mon.0) 434 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:00.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[79878]: cephadm 2024-03-07T03:17:59.467297+0000 mgr.y (mgr.34107) 141 : cephadm 1 Reconfiguring daemon osd.6 on smithi189 2024-03-07T03:18:00.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[82329]: cluster 2024-03-07T03:17:59.284358+0000 mon.a (mon.0) 430 : cluster 0 osdmap e430: 8 total, 8 up, 8 in 2024-03-07T03:18:00.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[82329]: audit 2024-03-07T03:17:59.440097+0000 mon.a (mon.0) 431 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:00.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[82329]: audit 2024-03-07T03:17:59.453665+0000 mon.a (mon.0) 432 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:00.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[82329]: cephadm 2024-03-07T03:17:59.458010+0000 mgr.y (mgr.34107) 140 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2024-03-07T03:18:00.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[82329]: audit 2024-03-07T03:17:59.458715+0000 mon.a (mon.0) 433 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-07T03:18:00.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[82329]: audit 2024-03-07T03:17:59.460888+0000 mon.a (mon.0) 434 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:00.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:00 smithi145 bash[82329]: cephadm 2024-03-07T03:17:59.467297+0000 mgr.y (mgr.34107) 141 : cephadm 1 Reconfiguring daemon osd.6 on smithi189 2024-03-07T03:18:00.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:00 smithi189 bash[66969]: cluster 2024-03-07T03:17:59.284358+0000 mon.a (mon.0) 430 : cluster 0 osdmap e430: 8 total, 8 up, 8 in 2024-03-07T03:18:00.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:00 smithi189 bash[66969]: audit 2024-03-07T03:17:59.440097+0000 mon.a (mon.0) 431 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:00.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:00 smithi189 bash[66969]: audit 2024-03-07T03:17:59.453665+0000 mon.a (mon.0) 432 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:00.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:00 smithi189 bash[66969]: cephadm 2024-03-07T03:17:59.458010+0000 mgr.y (mgr.34107) 140 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2024-03-07T03:18:00.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:00 smithi189 bash[66969]: audit 2024-03-07T03:17:59.458715+0000 mon.a (mon.0) 433 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-07T03:18:00.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:00 smithi189 bash[66969]: audit 2024-03-07T03:17:59.460888+0000 mon.a (mon.0) 434 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:00.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:00 smithi189 bash[66969]: cephadm 2024-03-07T03:17:59.467297+0000 mgr.y (mgr.34107) 141 : cephadm 1 Reconfiguring daemon osd.6 on smithi189 2024-03-07T03:18:00.658 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 17491264~6709117 2024-03-07T03:18:00.699 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 1844730~6731053 2024-03-07T03:18:00.723 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 32257350~3737583 2024-03-07T03:18:00.762 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 146908~1006179 2024-03-07T03:18:00.766 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 45886342~4413964 2024-03-07T03:18:00.794 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 14312083~624325 2024-03-07T03:18:00.800 INFO:tasks.workunit.client.0.smithi145.stdout:write: C 48779100~4158425 2024-03-07T03:18:00.817 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 4713687~3234039 2024-03-07T03:18:00.819 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap2 2024-03-07T03:18:01.486 INFO:tasks.workunit.client.0.smithi145.stdout:clone image284 -> image285 2024-03-07T03:18:01.665 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 159433606 2024-03-07T03:18:01.678 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 86711792~1349162 2024-03-07T03:18:01.694 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 46759700~981924 2024-03-07T03:18:01.730 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 156495873~2937733 2024-03-07T03:18:01.752 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 129150403~5183653 2024-03-07T03:18:01.754 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 150323455~5725510 2024-03-07T03:18:01.783 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 78037108~3107961 2024-03-07T03:18:01.794 INFO:tasks.workunit.client.0.smithi145.stdout:write: D 153345924~6087682 2024-03-07T03:18:01.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:01 smithi145 bash[79878]: cluster 2024-03-07T03:18:00.456429+0000 mon.a (mon.0) 435 : cluster 0 osdmap e431: 8 total, 8 up, 8 in 2024-03-07T03:18:01.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:01 smithi145 bash[79878]: cluster 2024-03-07T03:18:00.589230+0000 mgr.y (mgr.34107) 142 : cluster 0 pgmap v241: 105 pgs: 105 active+clean; 773 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 20 MiB/s wr, 538 op/s 2024-03-07T03:18:01.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:01 smithi145 bash[82329]: cluster 2024-03-07T03:18:00.456429+0000 mon.a (mon.0) 435 : cluster 0 osdmap e431: 8 total, 8 up, 8 in 2024-03-07T03:18:01.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:01 smithi145 bash[82329]: cluster 2024-03-07T03:18:00.589230+0000 mgr.y (mgr.34107) 142 : cluster 0 pgmap v241: 105 pgs: 105 active+clean; 773 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 20 MiB/s wr, 538 op/s 2024-03-07T03:18:01.816 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 49675236~1182405 2024-03-07T03:18:01.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:01 smithi189 bash[66969]: cluster 2024-03-07T03:18:00.456429+0000 mon.a (mon.0) 435 : cluster 0 osdmap e431: 8 total, 8 up, 8 in 2024-03-07T03:18:01.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:01 smithi189 bash[66969]: cluster 2024-03-07T03:18:00.589230+0000 mgr.y (mgr.34107) 142 : cluster 0 pgmap v241: 105 pgs: 105 active+clean; 773 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 20 MiB/s wr, 538 op/s 2024-03-07T03:18:01.841 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap3 2024-03-07T03:18:02.451 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:18:02 smithi189 bash[59814]: ts=2024-03-07T03:18:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:02.619 INFO:tasks.workunit.client.0.smithi145.stdout:resize: 159783373 2024-03-07T03:18:02.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[79878]: cluster 2024-03-07T03:18:01.467790+0000 mon.a (mon.0) 436 : cluster 0 osdmap e432: 8 total, 8 up, 8 in 2024-03-07T03:18:02.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[79878]: audit 2024-03-07T03:18:01.591782+0000 mon.a (mon.0) 437 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:02.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[79878]: audit 2024-03-07T03:18:01.604519+0000 mon.a (mon.0) 438 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:02.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[79878]: cephadm 2024-03-07T03:18:01.635784+0000 mgr.y (mgr.34107) 143 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2024-03-07T03:18:02.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[79878]: audit 2024-03-07T03:18:01.636526+0000 mon.a (mon.0) 439 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-07T03:18:02.752 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[79878]: audit 2024-03-07T03:18:01.638425+0000 mon.a (mon.0) 440 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:02.752 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[79878]: cephadm 2024-03-07T03:18:01.643852+0000 mgr.y (mgr.34107) 144 : cephadm 1 Reconfiguring daemon osd.7 on smithi189 2024-03-07T03:18:02.752 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[82329]: cluster 2024-03-07T03:18:01.467790+0000 mon.a (mon.0) 436 : cluster 0 osdmap e432: 8 total, 8 up, 8 in 2024-03-07T03:18:02.752 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[82329]: audit 2024-03-07T03:18:01.591782+0000 mon.a (mon.0) 437 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:02.752 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[82329]: audit 2024-03-07T03:18:01.604519+0000 mon.a (mon.0) 438 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:02.752 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[82329]: cephadm 2024-03-07T03:18:01.635784+0000 mgr.y (mgr.34107) 143 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2024-03-07T03:18:02.752 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[82329]: audit 2024-03-07T03:18:01.636526+0000 mon.a (mon.0) 439 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-07T03:18:02.752 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[82329]: audit 2024-03-07T03:18:01.638425+0000 mon.a (mon.0) 440 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:02.752 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[82329]: cephadm 2024-03-07T03:18:01.643852+0000 mgr.y (mgr.34107) 144 : cephadm 1 Reconfiguring daemon osd.7 on smithi189 2024-03-07T03:18:02.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:02 smithi189 bash[66969]: cluster 2024-03-07T03:18:01.467790+0000 mon.a (mon.0) 436 : cluster 0 osdmap e432: 8 total, 8 up, 8 in 2024-03-07T03:18:02.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:02 smithi189 bash[66969]: audit 2024-03-07T03:18:01.591782+0000 mon.a (mon.0) 437 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:02.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:02 smithi189 bash[66969]: audit 2024-03-07T03:18:01.604519+0000 mon.a (mon.0) 438 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:02.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:02 smithi189 bash[66969]: cephadm 2024-03-07T03:18:01.635784+0000 mgr.y (mgr.34107) 143 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2024-03-07T03:18:02.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:02 smithi189 bash[66969]: audit 2024-03-07T03:18:01.636526+0000 mon.a (mon.0) 439 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-07T03:18:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:02 smithi189 bash[66969]: audit 2024-03-07T03:18:01.638425+0000 mon.a (mon.0) 440 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:02.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:02 smithi189 bash[66969]: cephadm 2024-03-07T03:18:01.643852+0000 mgr.y (mgr.34107) 144 : cephadm 1 Reconfiguring daemon osd.7 on smithi189 2024-03-07T03:18:03.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:18:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:18:02] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.43.0" 2024-03-07T03:18:03.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:18:03 smithi189 bash[59814]: ts=2024-03-07T03:18:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:03 smithi145 bash[79878]: cluster 2024-03-07T03:18:02.589939+0000 mgr.y (mgr.34107) 145 : cluster 0 pgmap v243: 105 pgs: 105 active+clean; 773 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s rd, 19 MiB/s wr, 498 op/s 2024-03-07T03:18:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:03 smithi145 bash[79878]: cluster 2024-03-07T03:18:02.609967+0000 mon.a (mon.0) 441 : cluster 0 osdmap e433: 8 total, 8 up, 8 in 2024-03-07T03:18:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:03 smithi145 bash[79878]: audit 2024-03-07T03:18:02.635696+0000 mon.a (mon.0) 442 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:04.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:03 smithi145 bash[82329]: cluster 2024-03-07T03:18:02.589939+0000 mgr.y (mgr.34107) 145 : cluster 0 pgmap v243: 105 pgs: 105 active+clean; 773 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s rd, 19 MiB/s wr, 498 op/s 2024-03-07T03:18:04.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:03 smithi145 bash[82329]: cluster 2024-03-07T03:18:02.609967+0000 mon.a (mon.0) 441 : cluster 0 osdmap e433: 8 total, 8 up, 8 in 2024-03-07T03:18:04.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:03 smithi145 bash[82329]: audit 2024-03-07T03:18:02.635696+0000 mon.a (mon.0) 442 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:03 smithi189 bash[66969]: cluster 2024-03-07T03:18:02.589939+0000 mgr.y (mgr.34107) 145 : cluster 0 pgmap v243: 105 pgs: 105 active+clean; 773 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s rd, 19 MiB/s wr, 498 op/s 2024-03-07T03:18:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:03 smithi189 bash[66969]: cluster 2024-03-07T03:18:02.609967+0000 mon.a (mon.0) 441 : cluster 0 osdmap e433: 8 total, 8 up, 8 in 2024-03-07T03:18:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:03 smithi189 bash[66969]: audit 2024-03-07T03:18:02.635696+0000 mon.a (mon.0) 442 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:04.325 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestDeepCopy.Stress_SmallerDstObjSize (5428 ms) 2024-03-07T03:18:04.326 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize 2024-03-07T03:18:04.383 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_DeepCopy.cc:621: Skipped 2024-03-07T03:18:04.383 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:18:04.538 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize (213 ms) 2024-03-07T03:18:04.538 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingLargerDstObjSize 2024-03-07T03:18:04.604 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_DeepCopy.cc:635: Skipped 2024-03-07T03:18:04.604 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:18:04.786 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.Snaps_StrippingLargerDstObjSize (248 ms) 2024-03-07T03:18:04.786 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Clone_StrippingLargerDstObjSize 2024-03-07T03:18:04.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:04 smithi189 bash[66969]: cluster 2024-03-07T03:18:03.618682+0000 mon.a (mon.0) 443 : cluster 0 osdmap e434: 8 total, 8 up, 8 in 2024-03-07T03:18:04.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:04 smithi189 bash[66969]: audit 2024-03-07T03:18:03.895518+0000 mon.a (mon.0) 444 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:04.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:04 smithi189 bash[66969]: audit 2024-03-07T03:18:03.908777+0000 mon.a (mon.0) 445 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:04.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:04 smithi189 bash[66969]: audit 2024-03-07T03:18:03.919693+0000 mon.a (mon.0) 446 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.zkinbe", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:18:04.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:04 smithi189 bash[66969]: audit 2024-03-07T03:18:03.921768+0000 mon.a (mon.0) 447 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:04.851 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_DeepCopy.cc:649: Skipped 2024-03-07T03:18:04.851 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:18:05.026 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.Clone_StrippingLargerDstObjSize (240 ms) 2024-03-07T03:18:05.026 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize 2024-03-07T03:18:05.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:04 smithi145 bash[79878]: cluster 2024-03-07T03:18:03.618682+0000 mon.a (mon.0) 443 : cluster 0 osdmap e434: 8 total, 8 up, 8 in 2024-03-07T03:18:05.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:04 smithi145 bash[79878]: audit 2024-03-07T03:18:03.895518+0000 mon.a (mon.0) 444 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:05.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:04 smithi145 bash[79878]: audit 2024-03-07T03:18:03.908777+0000 mon.a (mon.0) 445 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:05.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:04 smithi145 bash[79878]: audit 2024-03-07T03:18:03.919693+0000 mon.a (mon.0) 446 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.zkinbe", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:18:05.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:04 smithi145 bash[79878]: audit 2024-03-07T03:18:03.921768+0000 mon.a (mon.0) 447 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:04 smithi145 bash[82329]: cluster 2024-03-07T03:18:03.618682+0000 mon.a (mon.0) 443 : cluster 0 osdmap e434: 8 total, 8 up, 8 in 2024-03-07T03:18:05.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:04 smithi145 bash[82329]: audit 2024-03-07T03:18:03.895518+0000 mon.a (mon.0) 444 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:05.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:04 smithi145 bash[82329]: audit 2024-03-07T03:18:03.908777+0000 mon.a (mon.0) 445 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:05.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:04 smithi145 bash[82329]: audit 2024-03-07T03:18:03.919693+0000 mon.a (mon.0) 446 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi189.zkinbe", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:18:05.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:04 smithi145 bash[82329]: audit 2024-03-07T03:18:03.921768+0000 mon.a (mon.0) 447 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:05.087 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_DeepCopy.cc:663: Skipped 2024-03-07T03:18:05.087 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:18:05.256 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize (229 ms) 2024-03-07T03:18:05.256 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Stress_StrippingLargerDstObjSize 2024-03-07T03:18:05.317 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_DeepCopy.cc:679: Skipped 2024-03-07T03:18:05.317 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:18:05.471 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.Stress_StrippingLargerDstObjSize (216 ms) 2024-03-07T03:18:05.471 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize 2024-03-07T03:18:05.522 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_DeepCopy.cc:693: Skipped 2024-03-07T03:18:05.522 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:18:05.678 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize (206 ms) 2024-03-07T03:18:05.678 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize 2024-03-07T03:18:05.738 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_DeepCopy.cc:707: Skipped 2024-03-07T03:18:05.738 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:18:05.912 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize (234 ms) 2024-03-07T03:18:05.912 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Clone_StrippingSmallerDstObjSize 2024-03-07T03:18:05.976 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_DeepCopy.cc:721: Skipped 2024-03-07T03:18:05.976 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:18:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:05 smithi145 bash[79878]: cephadm 2024-03-07T03:18:03.918892+0000 mgr.y (mgr.34107) 146 : cephadm 1 Reconfiguring mds.foo.smithi189.zkinbe (monmap changed)... 2024-03-07T03:18:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:05 smithi145 bash[79878]: cephadm 2024-03-07T03:18:03.924015+0000 mgr.y (mgr.34107) 147 : cephadm 1 Reconfiguring daemon mds.foo.smithi189.zkinbe on smithi189 2024-03-07T03:18:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:05 smithi145 bash[79878]: cluster 2024-03-07T03:18:04.590767+0000 mgr.y (mgr.34107) 148 : cluster 0 pgmap v246: 105 pgs: 105 active+clean; 963 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 38 MiB/s wr, 667 op/s 2024-03-07T03:18:06.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:05 smithi145 bash[82329]: cephadm 2024-03-07T03:18:03.918892+0000 mgr.y (mgr.34107) 146 : cephadm 1 Reconfiguring mds.foo.smithi189.zkinbe (monmap changed)... 2024-03-07T03:18:06.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:05 smithi145 bash[82329]: cephadm 2024-03-07T03:18:03.924015+0000 mgr.y (mgr.34107) 147 : cephadm 1 Reconfiguring daemon mds.foo.smithi189.zkinbe on smithi189 2024-03-07T03:18:06.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:05 smithi145 bash[82329]: cluster 2024-03-07T03:18:04.590767+0000 mgr.y (mgr.34107) 148 : cluster 0 pgmap v246: 105 pgs: 105 active+clean; 963 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 38 MiB/s wr, 667 op/s 2024-03-07T03:18:06.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:05 smithi189 bash[66969]: cephadm 2024-03-07T03:18:03.918892+0000 mgr.y (mgr.34107) 146 : cephadm 1 Reconfiguring mds.foo.smithi189.zkinbe (monmap changed)... 2024-03-07T03:18:06.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:05 smithi189 bash[66969]: cephadm 2024-03-07T03:18:03.924015+0000 mgr.y (mgr.34107) 147 : cephadm 1 Reconfiguring daemon mds.foo.smithi189.zkinbe on smithi189 2024-03-07T03:18:06.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:05 smithi189 bash[66969]: cluster 2024-03-07T03:18:04.590767+0000 mgr.y (mgr.34107) 148 : cluster 0 pgmap v246: 105 pgs: 105 active+clean; 963 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 38 MiB/s wr, 667 op/s 2024-03-07T03:18:06.148 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.Clone_StrippingSmallerDstObjSize (237 ms) 2024-03-07T03:18:06.148 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize 2024-03-07T03:18:06.211 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_DeepCopy.cc:735: Skipped 2024-03-07T03:18:06.211 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:18:06.376 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize (228 ms) 2024-03-07T03:18:06.376 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestDeepCopy.Stress_StrippingSmallerDstObjSize 2024-03-07T03:18:06.439 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_DeepCopy.cc:751: Skipped 2024-03-07T03:18:06.439 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:18:06.599 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.Stress_StrippingSmallerDstObjSize (223 ms) 2024-03-07T03:18:07.215 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 31 tests from TestDeepCopy (63282 ms total) 2024-03-07T03:18:07.215 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:18:07.215 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 51 tests from TestMirroring 2024-03-07T03:18:07.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:07 smithi145 bash[79878]: audit 2024-03-07T03:18:06.181879+0000 mon.a (mon.0) 448 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:07.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:07 smithi145 bash[79878]: audit 2024-03-07T03:18:06.196278+0000 mon.a (mon.0) 449 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:07.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:07 smithi145 bash[79878]: audit 2024-03-07T03:18:06.259857+0000 mon.a (mon.0) 450 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:07.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:07 smithi145 bash[79878]: cluster 2024-03-07T03:18:06.591581+0000 mgr.y (mgr.34107) 149 : cluster 0 pgmap v247: 105 pgs: 105 active+clean; 963 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 26 MiB/s wr, 460 op/s 2024-03-07T03:18:07.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:07 smithi145 bash[82329]: audit 2024-03-07T03:18:06.181879+0000 mon.a (mon.0) 448 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:07.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:07 smithi145 bash[82329]: audit 2024-03-07T03:18:06.196278+0000 mon.a (mon.0) 449 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:07.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:07 smithi145 bash[82329]: audit 2024-03-07T03:18:06.259857+0000 mon.a (mon.0) 450 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:07.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:07 smithi145 bash[82329]: cluster 2024-03-07T03:18:06.591581+0000 mgr.y (mgr.34107) 149 : cluster 0 pgmap v247: 105 pgs: 105 active+clean; 963 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 26 MiB/s wr, 460 op/s 2024-03-07T03:18:07.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:07 smithi189 bash[66969]: audit 2024-03-07T03:18:06.181879+0000 mon.a (mon.0) 448 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:07.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:07 smithi189 bash[66969]: audit 2024-03-07T03:18:06.196278+0000 mon.a (mon.0) 449 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:07.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:07 smithi189 bash[66969]: audit 2024-03-07T03:18:06.259857+0000 mon.a (mon.0) 450 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:07.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:07 smithi189 bash[66969]: cluster 2024-03-07T03:18:06.591581+0000 mgr.y (mgr.34107) 149 : cluster 0 pgmap v247: 105 pgs: 105 active+clean; 963 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 MiB/s rd, 26 MiB/s wr, 460 op/s 2024-03-07T03:18:08.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:08 smithi145 bash[79878]: cluster 2024-03-07T03:18:07.209261+0000 mon.a (mon.0) 451 : cluster 0 osdmap e435: 8 total, 8 up, 8 in 2024-03-07T03:18:08.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:08 smithi145 bash[82329]: cluster 2024-03-07T03:18:07.209261+0000 mon.a (mon.0) 451 : cluster 0 osdmap e435: 8 total, 8 up, 8 in 2024-03-07T03:18:08.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:08 smithi189 bash[66969]: cluster 2024-03-07T03:18:07.209261+0000 mon.a (mon.0) 451 : cluster 0 osdmap e435: 8 total, 8 up, 8 in 2024-03-07T03:18:09.225 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage 2024-03-07T03:18:09.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:09 smithi145 bash[79878]: cluster 2024-03-07T03:18:08.219244+0000 mon.a (mon.0) 452 : cluster 0 osdmap e436: 8 total, 8 up, 8 in 2024-03-07T03:18:09.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:09 smithi145 bash[79878]: audit 2024-03-07T03:18:08.221311+0000 mon.a (mon.0) 453 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:18:09.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:09 smithi145 bash[79878]: cluster 2024-03-07T03:18:08.592855+0000 mgr.y (mgr.34107) 150 : cluster 0 pgmap v250: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:18:09.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:09 smithi145 bash[82329]: cluster 2024-03-07T03:18:08.219244+0000 mon.a (mon.0) 452 : cluster 0 osdmap e436: 8 total, 8 up, 8 in 2024-03-07T03:18:09.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:09 smithi145 bash[82329]: audit 2024-03-07T03:18:08.221311+0000 mon.a (mon.0) 453 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:18:09.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:09 smithi145 bash[82329]: cluster 2024-03-07T03:18:08.592855+0000 mgr.y (mgr.34107) 150 : cluster 0 pgmap v250: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:18:09.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:09 smithi189 bash[66969]: cluster 2024-03-07T03:18:08.219244+0000 mon.a (mon.0) 452 : cluster 0 osdmap e436: 8 total, 8 up, 8 in 2024-03-07T03:18:09.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:09 smithi189 bash[66969]: audit 2024-03-07T03:18:08.221311+0000 mon.a (mon.0) 453 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:18:09.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:09 smithi189 bash[66969]: cluster 2024-03-07T03:18:08.592855+0000 mgr.y (mgr.34107) 150 : cluster 0 pgmap v250: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:18:10.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:10 smithi145 bash[79878]: cluster 2024-03-07T03:18:09.211925+0000 mon.a (mon.0) 454 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:18:10.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:10 smithi145 bash[79878]: audit 2024-03-07T03:18:09.224817+0000 mon.a (mon.0) 455 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:18:10.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:10 smithi145 bash[79878]: cluster 2024-03-07T03:18:09.235953+0000 mon.a (mon.0) 456 : cluster 0 osdmap e437: 8 total, 8 up, 8 in 2024-03-07T03:18:10.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:10 smithi145 bash[82329]: cluster 2024-03-07T03:18:09.211925+0000 mon.a (mon.0) 454 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:18:10.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:10 smithi145 bash[82329]: audit 2024-03-07T03:18:09.224817+0000 mon.a (mon.0) 455 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:18:10.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:10 smithi145 bash[82329]: cluster 2024-03-07T03:18:09.235953+0000 mon.a (mon.0) 456 : cluster 0 osdmap e437: 8 total, 8 up, 8 in 2024-03-07T03:18:10.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:10 smithi189 bash[66969]: cluster 2024-03-07T03:18:09.211925+0000 mon.a (mon.0) 454 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:18:10.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:10 smithi189 bash[66969]: audit 2024-03-07T03:18:09.224817+0000 mon.a (mon.0) 455 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:18:10.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:10 smithi189 bash[66969]: cluster 2024-03-07T03:18:09.235953+0000 mon.a (mon.0) 456 : cluster 0 osdmap e437: 8 total, 8 up, 8 in 2024-03-07T03:18:11.471 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage (2245 ms) 2024-03-07T03:18:11.471 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool 2024-03-07T03:18:11.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:11 smithi145 bash[79878]: cluster 2024-03-07T03:18:10.243376+0000 mon.a (mon.0) 457 : cluster 0 osdmap e438: 8 total, 8 up, 8 in 2024-03-07T03:18:11.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:11 smithi145 bash[79878]: cluster 2024-03-07T03:18:10.593554+0000 mgr.y (mgr.34107) 151 : cluster 0 pgmap v253: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:18:11.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:11 smithi145 bash[82329]: cluster 2024-03-07T03:18:10.243376+0000 mon.a (mon.0) 457 : cluster 0 osdmap e438: 8 total, 8 up, 8 in 2024-03-07T03:18:11.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:11 smithi145 bash[82329]: cluster 2024-03-07T03:18:10.593554+0000 mgr.y (mgr.34107) 151 : cluster 0 pgmap v253: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:18:11.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:11 smithi189 bash[66969]: cluster 2024-03-07T03:18:10.243376+0000 mon.a (mon.0) 457 : cluster 0 osdmap e438: 8 total, 8 up, 8 in 2024-03-07T03:18:11.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:11 smithi189 bash[66969]: cluster 2024-03-07T03:18:10.593554+0000 mgr.y (mgr.34107) 151 : cluster 0 pgmap v253: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:18:11.660 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool (190 ms) 2024-03-07T03:18:11.660 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled 2024-03-07T03:18:11.788 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled (128 ms) 2024-03-07T03:18:11.788 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage 2024-03-07T03:18:11.952 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage (164 ms) 2024-03-07T03:18:11.952 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap 2024-03-07T03:18:12.140 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap (187 ms) 2024-03-07T03:18:12.140 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModePool 2024-03-07T03:18:12.328 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModePool (189 ms) 2024-03-07T03:18:12.328 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled 2024-03-07T03:18:12.554 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled (226 ms) 2024-03-07T03:18:12.554 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.DisableImageMirrorWithPeer 2024-03-07T03:18:12.554 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_mirroring.cc:482: Skipped 2024-03-07T03:18:12.554 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:18:12.554 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMirroring.DisableImageMirrorWithPeer (0 ms) 2024-03-07T03:18:12.554 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.DisableJournalingWithPeer 2024-03-07T03:18:12.554 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_mirroring.cc:520: Skipped 2024-03-07T03:18:12.555 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:18:12.555 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMirroring.DisableJournalingWithPeer (0 ms) 2024-03-07T03:18:12.555 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling 2024-03-07T03:18:12.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:12 smithi145 bash[79878]: cluster 2024-03-07T03:18:11.236277+0000 mon.a (mon.0) 458 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:18:12.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:12 smithi145 bash[79878]: cluster 2024-03-07T03:18:11.256577+0000 mon.a (mon.0) 459 : cluster 0 osdmap e439: 8 total, 8 up, 8 in 2024-03-07T03:18:12.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:12 smithi145 bash[79878]: audit 2024-03-07T03:18:12.097866+0000 mon.a (mon.0) 460 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:12.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:12 smithi145 bash[79878]: audit 2024-03-07T03:18:12.104401+0000 mon.a (mon.0) 461 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:12.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:12 smithi145 bash[82329]: cluster 2024-03-07T03:18:11.236277+0000 mon.a (mon.0) 458 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:18:12.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:12 smithi145 bash[82329]: cluster 2024-03-07T03:18:11.256577+0000 mon.a (mon.0) 459 : cluster 0 osdmap e439: 8 total, 8 up, 8 in 2024-03-07T03:18:12.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:12 smithi145 bash[82329]: audit 2024-03-07T03:18:12.097866+0000 mon.a (mon.0) 460 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:12.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:12 smithi145 bash[82329]: audit 2024-03-07T03:18:12.104401+0000 mon.a (mon.0) 461 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:18:12 smithi189 bash[59814]: ts=2024-03-07T03:18:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:12.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:12 smithi189 bash[66969]: cluster 2024-03-07T03:18:11.236277+0000 mon.a (mon.0) 458 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:18:12.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:12 smithi189 bash[66969]: cluster 2024-03-07T03:18:11.256577+0000 mon.a (mon.0) 459 : cluster 0 osdmap e439: 8 total, 8 up, 8 in 2024-03-07T03:18:12.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:12 smithi189 bash[66969]: audit 2024-03-07T03:18:12.097866+0000 mon.a (mon.0) 460 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:12.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:12 smithi189 bash[66969]: audit 2024-03-07T03:18:12.104401+0000 mon.a (mon.0) 461 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:12.678 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling (124 ms) 2024-03-07T03:18:12.678 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling 2024-03-07T03:18:12.796 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling (117 ms) 2024-03-07T03:18:12.796 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling 2024-03-07T03:18:13.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:18:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:18:12] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.43.0" 2024-03-07T03:18:13.496 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:13 smithi189 bash[66969]: cluster 2024-03-07T03:18:12.593998+0000 mgr.y (mgr.34107) 152 : cluster 0 pgmap v255: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:18:13.496 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:13 smithi189 bash[66969]: audit 2024-03-07T03:18:12.673533+0000 mon.a (mon.0) 462 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:13.496 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:13 smithi189 bash[66969]: audit 2024-03-07T03:18:12.674557+0000 mon.a (mon.0) 463 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:18:13.496 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:13 smithi189 bash[66969]: audit 2024-03-07T03:18:12.682114+0000 mon.a (mon.0) 464 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:13.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:13 smithi145 bash[79878]: cluster 2024-03-07T03:18:12.593998+0000 mgr.y (mgr.34107) 152 : cluster 0 pgmap v255: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:18:13.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:13 smithi145 bash[79878]: audit 2024-03-07T03:18:12.673533+0000 mon.a (mon.0) 462 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:13.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:13 smithi145 bash[79878]: audit 2024-03-07T03:18:12.674557+0000 mon.a (mon.0) 463 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:18:13.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:13 smithi145 bash[79878]: audit 2024-03-07T03:18:12.682114+0000 mon.a (mon.0) 464 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:13.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:13 smithi145 bash[82329]: cluster 2024-03-07T03:18:12.593998+0000 mgr.y (mgr.34107) 152 : cluster 0 pgmap v255: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:18:13.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:13 smithi145 bash[82329]: audit 2024-03-07T03:18:12.673533+0000 mon.a (mon.0) 462 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:13.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:13 smithi145 bash[82329]: audit 2024-03-07T03:18:12.674557+0000 mon.a (mon.0) 463 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:18:13.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:13 smithi145 bash[82329]: audit 2024-03-07T03:18:12.682114+0000 mon.a (mon.0) 464 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:13.829 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:18:13 smithi189 bash[59814]: ts=2024-03-07T03:18:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:14.797 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling (2001 ms) 2024-03-07T03:18:14.797 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock 2024-03-07T03:18:15.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:14 smithi145 bash[79878]: cluster 2024-03-07T03:18:13.688577+0000 mon.a (mon.0) 465 : cluster 0 osdmap e440: 8 total, 8 up, 8 in 2024-03-07T03:18:15.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:14 smithi145 bash[82329]: cluster 2024-03-07T03:18:13.688577+0000 mon.a (mon.0) 465 : cluster 0 osdmap e440: 8 total, 8 up, 8 in 2024-03-07T03:18:15.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:14 smithi189 bash[66969]: cluster 2024-03-07T03:18:13.688577+0000 mon.a (mon.0) 465 : cluster 0 osdmap e440: 8 total, 8 up, 8 in 2024-03-07T03:18:16.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:15 smithi145 bash[79878]: cluster 2024-03-07T03:18:14.594685+0000 mgr.y (mgr.34107) 153 : cluster 0 pgmap v257: 105 pgs: 105 active+clean; 582 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 271 KiB/s rd, 37 KiB/s wr, 393 op/s 2024-03-07T03:18:16.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:15 smithi145 bash[79878]: cluster 2024-03-07T03:18:14.698269+0000 mon.a (mon.0) 466 : cluster 0 osdmap e441: 8 total, 8 up, 8 in 2024-03-07T03:18:16.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:15 smithi145 bash[82329]: cluster 2024-03-07T03:18:14.594685+0000 mgr.y (mgr.34107) 153 : cluster 0 pgmap v257: 105 pgs: 105 active+clean; 582 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 271 KiB/s rd, 37 KiB/s wr, 393 op/s 2024-03-07T03:18:16.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:15 smithi145 bash[82329]: cluster 2024-03-07T03:18:14.698269+0000 mon.a (mon.0) 466 : cluster 0 osdmap e441: 8 total, 8 up, 8 in 2024-03-07T03:18:16.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:15 smithi189 bash[66969]: cluster 2024-03-07T03:18:14.594685+0000 mgr.y (mgr.34107) 153 : cluster 0 pgmap v257: 105 pgs: 105 active+clean; 582 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 271 KiB/s rd, 37 KiB/s wr, 393 op/s 2024-03-07T03:18:16.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:15 smithi189 bash[66969]: cluster 2024-03-07T03:18:14.698269+0000 mon.a (mon.0) 466 : cluster 0 osdmap e441: 8 total, 8 up, 8 in 2024-03-07T03:18:16.811 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock (2015 ms) 2024-03-07T03:18:16.811 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeDisabled 2024-03-07T03:18:17.047 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeDisabled (236 ms) 2024-03-07T03:18:17.048 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage 2024-03-07T03:18:17.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:16 smithi145 bash[79878]: cluster 2024-03-07T03:18:15.708880+0000 mon.a (mon.0) 467 : cluster 0 osdmap e442: 8 total, 8 up, 8 in 2024-03-07T03:18:17.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:16 smithi145 bash[82329]: cluster 2024-03-07T03:18:15.708880+0000 mon.a (mon.0) 467 : cluster 0 osdmap e442: 8 total, 8 up, 8 in 2024-03-07T03:18:17.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:16 smithi189 bash[66969]: cluster 2024-03-07T03:18:15.708880+0000 mon.a (mon.0) 467 : cluster 0 osdmap e442: 8 total, 8 up, 8 in 2024-03-07T03:18:17.306 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage (259 ms) 2024-03-07T03:18:17.306 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool 2024-03-07T03:18:17.523 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool (216 ms) 2024-03-07T03:18:17.523 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling 2024-03-07T03:18:17.701 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling (178 ms) 2024-03-07T03:18:17.701 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling 2024-03-07T03:18:17.928 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling (228 ms) 2024-03-07T03:18:17.928 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeDisabled 2024-03-07T03:18:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:17 smithi145 bash[82329]: cluster 2024-03-07T03:18:16.595412+0000 mgr.y (mgr.34107) 154 : cluster 0 pgmap v260: 105 pgs: 105 active+clean; 582 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 272 KiB/s rd, 37 KiB/s wr, 394 op/s 2024-03-07T03:18:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:17 smithi145 bash[82329]: cluster 2024-03-07T03:18:16.722004+0000 mon.a (mon.0) 468 : cluster 0 osdmap e443: 8 total, 8 up, 8 in 2024-03-07T03:18:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:17 smithi145 bash[82329]: cluster 2024-03-07T03:18:17.293823+0000 mon.a (mon.0) 469 : cluster 0 osdmap e444: 8 total, 8 up, 8 in 2024-03-07T03:18:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:17 smithi145 bash[82329]: audit 2024-03-07T03:18:17.636449+0000 mon.a (mon.0) 470 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:17 smithi145 bash[79878]: cluster 2024-03-07T03:18:16.595412+0000 mgr.y (mgr.34107) 154 : cluster 0 pgmap v260: 105 pgs: 105 active+clean; 582 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 272 KiB/s rd, 37 KiB/s wr, 394 op/s 2024-03-07T03:18:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:17 smithi145 bash[79878]: cluster 2024-03-07T03:18:16.722004+0000 mon.a (mon.0) 468 : cluster 0 osdmap e443: 8 total, 8 up, 8 in 2024-03-07T03:18:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:17 smithi145 bash[79878]: cluster 2024-03-07T03:18:17.293823+0000 mon.a (mon.0) 469 : cluster 0 osdmap e444: 8 total, 8 up, 8 in 2024-03-07T03:18:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:17 smithi145 bash[79878]: audit 2024-03-07T03:18:17.636449+0000 mon.a (mon.0) 470 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:18.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:17 smithi189 bash[66969]: cluster 2024-03-07T03:18:16.595412+0000 mgr.y (mgr.34107) 154 : cluster 0 pgmap v260: 105 pgs: 105 active+clean; 582 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 272 KiB/s rd, 37 KiB/s wr, 394 op/s 2024-03-07T03:18:18.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:17 smithi189 bash[66969]: cluster 2024-03-07T03:18:16.722004+0000 mon.a (mon.0) 468 : cluster 0 osdmap e443: 8 total, 8 up, 8 in 2024-03-07T03:18:18.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:17 smithi189 bash[66969]: cluster 2024-03-07T03:18:17.293823+0000 mon.a (mon.0) 469 : cluster 0 osdmap e444: 8 total, 8 up, 8 in 2024-03-07T03:18:18.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:17 smithi189 bash[66969]: audit 2024-03-07T03:18:17.636449+0000 mon.a (mon.0) 470 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:18.191 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeDisabled (263 ms) 2024-03-07T03:18:18.191 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage 2024-03-07T03:18:18.454 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage (263 ms) 2024-03-07T03:18:18.454 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled 2024-03-07T03:18:19.866 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled (1411 ms) 2024-03-07T03:18:19.866 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModePool 2024-03-07T03:18:20.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:19 smithi145 bash[79878]: cluster 2024-03-07T03:18:18.596721+0000 mgr.y (mgr.34107) 155 : cluster 0 pgmap v263: 105 pgs: 105 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail; 355 KiB/s rd, 41 KiB/s wr, 518 op/s 2024-03-07T03:18:20.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:19 smithi145 bash[79878]: cluster 2024-03-07T03:18:18.729993+0000 mon.a (mon.0) 471 : cluster 0 osdmap e445: 8 total, 8 up, 8 in 2024-03-07T03:18:20.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:19 smithi145 bash[82329]: cluster 2024-03-07T03:18:18.596721+0000 mgr.y (mgr.34107) 155 : cluster 0 pgmap v263: 105 pgs: 105 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail; 355 KiB/s rd, 41 KiB/s wr, 518 op/s 2024-03-07T03:18:20.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:19 smithi145 bash[82329]: cluster 2024-03-07T03:18:18.729993+0000 mon.a (mon.0) 471 : cluster 0 osdmap e445: 8 total, 8 up, 8 in 2024-03-07T03:18:20.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:19 smithi189 bash[66969]: cluster 2024-03-07T03:18:18.596721+0000 mgr.y (mgr.34107) 155 : cluster 0 pgmap v263: 105 pgs: 105 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail; 355 KiB/s rd, 41 KiB/s wr, 518 op/s 2024-03-07T03:18:20.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:19 smithi189 bash[66969]: cluster 2024-03-07T03:18:18.729993+0000 mon.a (mon.0) 471 : cluster 0 osdmap e445: 8 total, 8 up, 8 in 2024-03-07T03:18:20.124 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModePool (259 ms) 2024-03-07T03:18:20.124 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModePool 2024-03-07T03:18:20.379 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModePool (255 ms) 2024-03-07T03:18:20.379 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModeImage 2024-03-07T03:18:20.649 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModeImage (270 ms) 2024-03-07T03:18:20.649 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode 2024-03-07T03:18:21.011 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode (361 ms) 2024-03-07T03:18:21.011 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode 2024-03-07T03:18:21.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:20 smithi145 bash[79878]: cluster 2024-03-07T03:18:19.738861+0000 mon.a (mon.0) 472 : cluster 0 osdmap e446: 8 total, 8 up, 8 in 2024-03-07T03:18:21.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:20 smithi145 bash[82329]: cluster 2024-03-07T03:18:19.738861+0000 mon.a (mon.0) 472 : cluster 0 osdmap e446: 8 total, 8 up, 8 in 2024-03-07T03:18:21.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:20 smithi189 bash[66969]: cluster 2024-03-07T03:18:19.738861+0000 mon.a (mon.0) 472 : cluster 0 osdmap e446: 8 total, 8 up, 8 in 2024-03-07T03:18:21.395 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode (385 ms) 2024-03-07T03:18:21.395 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode 2024-03-07T03:18:21.778 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode (383 ms) 2024-03-07T03:18:21.778 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode 2024-03-07T03:18:22.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:21 smithi145 bash[79878]: cluster 2024-03-07T03:18:20.597432+0000 mgr.y (mgr.34107) 156 : cluster 0 pgmap v266: 105 pgs: 105 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail; 355 KiB/s rd, 41 KiB/s wr, 518 op/s 2024-03-07T03:18:22.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:21 smithi145 bash[82329]: cluster 2024-03-07T03:18:20.597432+0000 mgr.y (mgr.34107) 156 : cluster 0 pgmap v266: 105 pgs: 105 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail; 355 KiB/s rd, 41 KiB/s wr, 518 op/s 2024-03-07T03:18:22.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:21 smithi189 bash[66969]: cluster 2024-03-07T03:18:20.597432+0000 mgr.y (mgr.34107) 156 : cluster 0 pgmap v266: 105 pgs: 105 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail; 355 KiB/s rd, 41 KiB/s wr, 518 op/s 2024-03-07T03:18:22.102 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode (324 ms) 2024-03-07T03:18:22.102 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode 2024-03-07T03:18:22.438 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode (335 ms) 2024-03-07T03:18:22.438 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode 2024-03-07T03:18:22.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:18:22 smithi189 bash[59814]: ts=2024-03-07T03:18:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:22.798 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode (361 ms) 2024-03-07T03:18:22.798 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageEnabled 2024-03-07T03:18:23.021 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageEnabled (222 ms) 2024-03-07T03:18:23.021 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDisabled 2024-03-07T03:18:23.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:18:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:18:22] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.43.0" 2024-03-07T03:18:23.203 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDisabled (183 ms) 2024-03-07T03:18:23.203 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.RemoveImage_With_ImageWithoutJournal 2024-03-07T03:18:23.347 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade resume' 2024-03-07T03:18:23.366 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.RemoveImage_With_ImageWithoutJournal (163 ms) 2024-03-07T03:18:23.366 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDemoted 2024-03-07T03:18:23.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:23 smithi145 bash[79878]: cluster 2024-03-07T03:18:22.288347+0000 mon.a (mon.0) 473 : cluster 0 osdmap e447: 8 total, 8 up, 8 in 2024-03-07T03:18:23.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:23 smithi145 bash[79878]: cluster 2024-03-07T03:18:22.597904+0000 mgr.y (mgr.34107) 157 : cluster 0 pgmap v268: 105 pgs: 105 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail; 267 KiB/s rd, 31 KiB/s wr, 390 op/s 2024-03-07T03:18:23.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:23 smithi145 bash[82329]: cluster 2024-03-07T03:18:22.288347+0000 mon.a (mon.0) 473 : cluster 0 osdmap e447: 8 total, 8 up, 8 in 2024-03-07T03:18:23.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:23 smithi145 bash[82329]: cluster 2024-03-07T03:18:22.597904+0000 mgr.y (mgr.34107) 157 : cluster 0 pgmap v268: 105 pgs: 105 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail; 267 KiB/s rd, 31 KiB/s wr, 390 op/s 2024-03-07T03:18:23.570 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDemoted (204 ms) 2024-03-07T03:18:23.570 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.TrashMoveRestore_PoolMode 2024-03-07T03:18:23.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:18:23 smithi189 bash[59814]: ts=2024-03-07T03:18:23.516Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi145\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi145\", job=\"node\", machine=\"x86_64\", nodename=\"smithi145\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:23.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:23 smithi189 bash[66969]: cluster 2024-03-07T03:18:22.288347+0000 mon.a (mon.0) 473 : cluster 0 osdmap e447: 8 total, 8 up, 8 in 2024-03-07T03:18:23.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:23 smithi189 bash[66969]: cluster 2024-03-07T03:18:22.597904+0000 mgr.y (mgr.34107) 157 : cluster 0 pgmap v268: 105 pgs: 105 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail; 267 KiB/s rd, 31 KiB/s wr, 390 op/s 2024-03-07T03:18:23.856 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.TrashMoveRestore_PoolMode (286 ms) 2024-03-07T03:18:23.856 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled 2024-03-07T03:18:24.124 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled (268 ms) 2024-03-07T03:18:24.124 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled 2024-03-07T03:18:24.420 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled (296 ms) 2024-03-07T03:18:24.421 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.MirrorStatusList 2024-03-07T03:18:24.584 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.MirrorStatusList (164 ms) 2024-03-07T03:18:24.585 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.RemoveBootstrapped 2024-03-07T03:18:24.776 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.RemoveBootstrapped (192 ms) 2024-03-07T03:18:24.776 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.AioPromoteDemote 2024-03-07T03:18:24.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:24 smithi145 bash[79878]: audit 2024-03-07T03:18:23.565513+0000 mon.a (mon.0) 474 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:24.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:24 smithi145 bash[82329]: audit 2024-03-07T03:18:23.565513+0000 mon.a (mon.0) 474 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:24.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:24 smithi189 bash[66969]: audit 2024-03-07T03:18:23.565513+0000 mon.a (mon.0) 474 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:25.399 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.AioPromoteDemote (623 ms) 2024-03-07T03:18:25.400 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.AioGetInfo 2024-03-07T03:18:25.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:25 smithi145 bash[79878]: cluster 2024-03-07T03:18:24.598716+0000 mgr.y (mgr.34107) 158 : cluster 0 pgmap v269: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail; 501 KiB/s rd, 61 KiB/s wr, 694 op/s 2024-03-07T03:18:25.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:25 smithi145 bash[82329]: cluster 2024-03-07T03:18:24.598716+0000 mgr.y (mgr.34107) 158 : cluster 0 pgmap v269: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail; 501 KiB/s rd, 61 KiB/s wr, 694 op/s 2024-03-07T03:18:25.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:25 smithi189 bash[66969]: cluster 2024-03-07T03:18:24.598716+0000 mgr.y (mgr.34107) 158 : cluster 0 pgmap v269: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail; 501 KiB/s rd, 61 KiB/s wr, 694 op/s 2024-03-07T03:18:25.987 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.AioGetInfo (588 ms) 2024-03-07T03:18:25.987 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.AioGetStatus 2024-03-07T03:18:26.437 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.AioGetStatus (450 ms) 2024-03-07T03:18:26.437 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.SiteName 2024-03-07T03:18:26.447 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.SiteName (10 ms) 2024-03-07T03:18:26.447 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.Bootstrap 2024-03-07T03:18:27.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[79878]: audit 2024-03-07T03:18:26.454269+0000 mon.a (mon.0) 475 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' 2024-03-07T03:18:27.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[79878]: audit 2024-03-07T03:18:26.454656+0000 mon.a (mon.0) 476 : audit 0 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:18:27.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[79878]: audit 2024-03-07T03:18:26.454973+0000 mon.a (mon.0) 477 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:18:27.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[79878]: audit 2024-03-07T03:18:26.459680+0000 mon.a (mon.0) 478 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-03-07T03:18:27.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[79878]: audit 2024-03-07T03:18:26.460019+0000 mon.a (mon.0) 479 : audit 0 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:18:27.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[79878]: cluster 2024-03-07T03:18:26.599357+0000 mgr.y (mgr.34107) 159 : cluster 0 pgmap v270: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail; 382 KiB/s rd, 47 KiB/s wr, 529 op/s 2024-03-07T03:18:27.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[79878]: cluster 2024-03-07T03:18:27.289259+0000 mon.a (mon.0) 480 : cluster 0 osdmap e448: 8 total, 8 up, 8 in 2024-03-07T03:18:27.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[82329]: audit 2024-03-07T03:18:26.454269+0000 mon.a (mon.0) 475 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' 2024-03-07T03:18:27.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[82329]: audit 2024-03-07T03:18:26.454656+0000 mon.a (mon.0) 476 : audit 0 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:18:27.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[82329]: audit 2024-03-07T03:18:26.454973+0000 mon.a (mon.0) 477 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:18:27.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[82329]: audit 2024-03-07T03:18:26.459680+0000 mon.a (mon.0) 478 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-03-07T03:18:27.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[82329]: audit 2024-03-07T03:18:26.460019+0000 mon.a (mon.0) 479 : audit 0 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:18:27.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[82329]: cluster 2024-03-07T03:18:26.599357+0000 mgr.y (mgr.34107) 159 : cluster 0 pgmap v270: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail; 382 KiB/s rd, 47 KiB/s wr, 529 op/s 2024-03-07T03:18:27.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:27 smithi145 bash[82329]: cluster 2024-03-07T03:18:27.289259+0000 mon.a (mon.0) 480 : cluster 0 osdmap e448: 8 total, 8 up, 8 in 2024-03-07T03:18:27.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:27 smithi189 bash[66969]: audit 2024-03-07T03:18:26.454269+0000 mon.a (mon.0) 475 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' 2024-03-07T03:18:27.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:27 smithi189 bash[66969]: audit 2024-03-07T03:18:26.454656+0000 mon.a (mon.0) 476 : audit 0 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:18:27.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:27 smithi189 bash[66969]: audit 2024-03-07T03:18:26.454973+0000 mon.a (mon.0) 477 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:18:27.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:27 smithi189 bash[66969]: audit 2024-03-07T03:18:26.459680+0000 mon.a (mon.0) 478 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-03-07T03:18:27.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:27 smithi189 bash[66969]: audit 2024-03-07T03:18:26.460019+0000 mon.a (mon.0) 479 : audit 0 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:18:27.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:27 smithi189 bash[66969]: cluster 2024-03-07T03:18:26.599357+0000 mgr.y (mgr.34107) 159 : cluster 0 pgmap v270: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail; 382 KiB/s rd, 47 KiB/s wr, 529 op/s 2024-03-07T03:18:27.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:27 smithi189 bash[66969]: cluster 2024-03-07T03:18:27.289259+0000 mon.a (mon.0) 480 : cluster 0 osdmap e448: 8 total, 8 up, 8 in 2024-03-07T03:18:28.033 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.Bootstrap (1586 ms) 2024-03-07T03:18:28.034 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.PeerDirection 2024-03-07T03:18:28.050 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.PeerDirection (17 ms) 2024-03-07T03:18:28.050 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.Snapshot 2024-03-07T03:18:28.059 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:18:28.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[82329]: audit 2024-03-07T03:18:28.025245+0000 mon.a (mon.0) 481 : audit 0 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-03-07T03:18:28.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[82329]: audit 2024-03-07T03:18:28.029765+0000 mon.a (mon.0) 482 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' 2024-03-07T03:18:28.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[82329]: audit 2024-03-07T03:18:28.030286+0000 mon.a (mon.0) 483 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-03-07T03:18:28.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[82329]: audit 2024-03-07T03:18:28.033101+0000 mon.a (mon.0) 484 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-03-07T03:18:28.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[82329]: audit 2024-03-07T03:18:28.035432+0000 mon.a (mon.0) 485 : audit 0 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:18:28.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[82329]: audit 2024-03-07T03:18:28.054562+0000 mon.a (mon.0) 486 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0b2938a6-d6d6-4d0c-ad19-5c02dfe6f23f"}]: dispatch 2024-03-07T03:18:28.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[82329]: audit 2024-03-07T03:18:28.058463+0000 mon.a (mon.0) 487 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0b2938a6-d6d6-4d0c-ad19-5c02dfe6f23f"}]': finished 2024-03-07T03:18:28.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[79878]: audit 2024-03-07T03:18:28.025245+0000 mon.a (mon.0) 481 : audit 0 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-03-07T03:18:28.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[79878]: audit 2024-03-07T03:18:28.029765+0000 mon.a (mon.0) 482 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' 2024-03-07T03:18:28.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[79878]: audit 2024-03-07T03:18:28.030286+0000 mon.a (mon.0) 483 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-03-07T03:18:28.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[79878]: audit 2024-03-07T03:18:28.033101+0000 mon.a (mon.0) 484 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-03-07T03:18:28.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[79878]: audit 2024-03-07T03:18:28.035432+0000 mon.a (mon.0) 485 : audit 0 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:18:28.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[79878]: audit 2024-03-07T03:18:28.054562+0000 mon.a (mon.0) 486 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0b2938a6-d6d6-4d0c-ad19-5c02dfe6f23f"}]: dispatch 2024-03-07T03:18:28.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:28 smithi145 bash[79878]: audit 2024-03-07T03:18:28.058463+0000 mon.a (mon.0) 487 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0b2938a6-d6d6-4d0c-ad19-5c02dfe6f23f"}]': finished 2024-03-07T03:18:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:28 smithi189 bash[66969]: audit 2024-03-07T03:18:28.025245+0000 mon.a (mon.0) 481 : audit 0 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-03-07T03:18:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:28 smithi189 bash[66969]: audit 2024-03-07T03:18:28.029765+0000 mon.a (mon.0) 482 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' 2024-03-07T03:18:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:28 smithi189 bash[66969]: audit 2024-03-07T03:18:28.030286+0000 mon.a (mon.0) 483 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-03-07T03:18:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:28 smithi189 bash[66969]: audit 2024-03-07T03:18:28.033101+0000 mon.a (mon.0) 484 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-03-07T03:18:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:28 smithi189 bash[66969]: audit 2024-03-07T03:18:28.035432+0000 mon.a (mon.0) 485 : audit 0 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:18:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:28 smithi189 bash[66969]: audit 2024-03-07T03:18:28.054562+0000 mon.a (mon.0) 486 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0b2938a6-d6d6-4d0c-ad19-5c02dfe6f23f"}]: dispatch 2024-03-07T03:18:28.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:28 smithi189 bash[66969]: audit 2024-03-07T03:18:28.058463+0000 mon.a (mon.0) 487 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0b2938a6-d6d6-4d0c-ad19-5c02dfe6f23f"}]': finished 2024-03-07T03:18:29.726 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:29 smithi145 bash[79878]: cluster 2024-03-07T03:18:28.600609+0000 mgr.y (mgr.34107) 160 : cluster 0 pgmap v272: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 157 KiB/s wr, 1.45k op/s 2024-03-07T03:18:29.726 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:29 smithi145 bash[79878]: cluster 2024-03-07T03:18:29.066470+0000 mon.a (mon.0) 488 : cluster 0 osdmap e449: 8 total, 8 up, 8 in 2024-03-07T03:18:29.726 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:29 smithi145 bash[82329]: cluster 2024-03-07T03:18:28.600609+0000 mgr.y (mgr.34107) 160 : cluster 0 pgmap v272: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 157 KiB/s wr, 1.45k op/s 2024-03-07T03:18:29.726 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:29 smithi145 bash[82329]: cluster 2024-03-07T03:18:29.066470+0000 mon.a (mon.0) 488 : cluster 0 osdmap e449: 8 total, 8 up, 8 in 2024-03-07T03:18:29.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:29 smithi189 bash[66969]: cluster 2024-03-07T03:18:28.600609+0000 mgr.y (mgr.34107) 160 : cluster 0 pgmap v272: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 157 KiB/s wr, 1.45k op/s 2024-03-07T03:18:29.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:29 smithi189 bash[66969]: cluster 2024-03-07T03:18:29.066470+0000 mon.a (mon.0) 488 : cluster 0 osdmap e449: 8 total, 8 up, 8 in 2024-03-07T03:18:31.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:31 smithi189 bash[66969]: cluster 2024-03-07T03:18:30.069722+0000 mon.a (mon.0) 489 : cluster 0 osdmap e450: 8 total, 8 up, 8 in 2024-03-07T03:18:31.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:31 smithi189 bash[66969]: cluster 2024-03-07T03:18:30.601317+0000 mgr.y (mgr.34107) 161 : cluster 0 pgmap v275: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 911 KiB/s rd, 148 KiB/s wr, 1.23k op/s 2024-03-07T03:18:31.408 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:31 smithi145 bash[82329]: cluster 2024-03-07T03:18:30.069722+0000 mon.a (mon.0) 489 : cluster 0 osdmap e450: 8 total, 8 up, 8 in 2024-03-07T03:18:31.408 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:31 smithi145 bash[82329]: cluster 2024-03-07T03:18:30.601317+0000 mgr.y (mgr.34107) 161 : cluster 0 pgmap v275: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 911 KiB/s rd, 148 KiB/s wr, 1.23k op/s 2024-03-07T03:18:31.408 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:31 smithi145 bash[79878]: cluster 2024-03-07T03:18:30.069722+0000 mon.a (mon.0) 489 : cluster 0 osdmap e450: 8 total, 8 up, 8 in 2024-03-07T03:18:31.408 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:31 smithi145 bash[79878]: cluster 2024-03-07T03:18:30.601317+0000 mgr.y (mgr.34107) 161 : cluster 0 pgmap v275: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 911 KiB/s rd, 148 KiB/s wr, 1.23k op/s 2024-03-07T03:18:31.843 INFO:teuthology.orchestra.run.smithi145.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:18:32.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:32 smithi145 bash[79878]: cluster 2024-03-07T03:18:31.076645+0000 mon.a (mon.0) 490 : cluster 0 osdmap e451: 8 total, 8 up, 8 in 2024-03-07T03:18:32.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:32 smithi145 bash[79878]: audit 2024-03-07T03:18:31.855138+0000 mon.a (mon.0) 491 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:32.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:32 smithi145 bash[79878]: audit 2024-03-07T03:18:31.857896+0000 mon.a (mon.0) 492 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:32.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:32 smithi145 bash[82329]: cluster 2024-03-07T03:18:31.076645+0000 mon.a (mon.0) 490 : cluster 0 osdmap e451: 8 total, 8 up, 8 in 2024-03-07T03:18:32.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:32 smithi145 bash[82329]: audit 2024-03-07T03:18:31.855138+0000 mon.a (mon.0) 491 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:32.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:32 smithi145 bash[82329]: audit 2024-03-07T03:18:31.857896+0000 mon.a (mon.0) 492 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:32.329 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:18:32 smithi189 bash[59814]: ts=2024-03-07T03:18:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:32.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:32 smithi189 bash[66969]: cluster 2024-03-07T03:18:31.076645+0000 mon.a (mon.0) 490 : cluster 0 osdmap e451: 8 total, 8 up, 8 in 2024-03-07T03:18:32.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:32 smithi189 bash[66969]: audit 2024-03-07T03:18:31.855138+0000 mon.a (mon.0) 491 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:32.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:32 smithi189 bash[66969]: audit 2024-03-07T03:18:31.857896+0000 mon.a (mon.0) 492 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:32.592 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'echo wait for all mons to upgrade' 2024-03-07T03:18:33.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:18:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:18:32] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.43.0" 2024-03-07T03:18:33.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[79878]: audit 2024-03-07T03:18:31.849084+0000 mgr.y (mgr.34107) 162 : audit 0 from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:18:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[79878]: cephadm 2024-03-07T03:18:31.849567+0000 mgr.y (mgr.34107) 163 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:18:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[79878]: cluster 2024-03-07T03:18:32.087460+0000 mon.a (mon.0) 493 : cluster 0 osdmap e452: 8 total, 8 up, 8 in 2024-03-07T03:18:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[79878]: audit 2024-03-07T03:18:32.331031+0000 mon.a (mon.0) 494 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[79878]: audit 2024-03-07T03:18:32.339053+0000 mon.a (mon.0) 495 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[79878]: cluster 2024-03-07T03:18:32.601832+0000 mgr.y (mgr.34107) 164 : cluster 0 pgmap v278: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:18:33.630 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[79878]: audit 2024-03-07T03:18:32.636380+0000 mon.a (mon.0) 496 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:33.630 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[82329]: audit 2024-03-07T03:18:31.849084+0000 mgr.y (mgr.34107) 162 : audit 0 from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:18:33.630 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[82329]: cephadm 2024-03-07T03:18:31.849567+0000 mgr.y (mgr.34107) 163 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:18:33.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[82329]: cluster 2024-03-07T03:18:32.087460+0000 mon.a (mon.0) 493 : cluster 0 osdmap e452: 8 total, 8 up, 8 in 2024-03-07T03:18:33.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[82329]: audit 2024-03-07T03:18:32.331031+0000 mon.a (mon.0) 494 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:33.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[82329]: audit 2024-03-07T03:18:32.339053+0000 mon.a (mon.0) 495 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:33.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[82329]: cluster 2024-03-07T03:18:32.601832+0000 mgr.y (mgr.34107) 164 : cluster 0 pgmap v278: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:18:33.631 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:33 smithi145 bash[82329]: audit 2024-03-07T03:18:32.636380+0000 mon.a (mon.0) 496 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:33.631 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:33 smithi189 bash[66969]: audit 2024-03-07T03:18:31.849084+0000 mgr.y (mgr.34107) 162 : audit 0 from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:18:33.631 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:33 smithi189 bash[66969]: cephadm 2024-03-07T03:18:31.849567+0000 mgr.y (mgr.34107) 163 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:18:33.631 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:33 smithi189 bash[66969]: cluster 2024-03-07T03:18:32.087460+0000 mon.a (mon.0) 493 : cluster 0 osdmap e452: 8 total, 8 up, 8 in 2024-03-07T03:18:33.631 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:33 smithi189 bash[66969]: audit 2024-03-07T03:18:32.331031+0000 mon.a (mon.0) 494 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:33.632 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:33 smithi189 bash[66969]: audit 2024-03-07T03:18:32.339053+0000 mon.a (mon.0) 495 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:33.632 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:33 smithi189 bash[66969]: cluster 2024-03-07T03:18:32.601832+0000 mgr.y (mgr.34107) 164 : cluster 0 pgmap v278: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:18:33.632 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:33 smithi189 bash[66969]: audit 2024-03-07T03:18:32.636380+0000 mon.a (mon.0) 496 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:34.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:34 smithi145 bash[79878]: cluster 2024-03-07T03:18:33.091615+0000 mon.a (mon.0) 497 : cluster 0 osdmap e453: 8 total, 8 up, 8 in 2024-03-07T03:18:34.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:34 smithi145 bash[82329]: cluster 2024-03-07T03:18:33.091615+0000 mon.a (mon.0) 497 : cluster 0 osdmap e453: 8 total, 8 up, 8 in 2024-03-07T03:18:34.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:34 smithi189 bash[66969]: cluster 2024-03-07T03:18:33.091615+0000 mon.a (mon.0) 497 : cluster 0 osdmap e453: 8 total, 8 up, 8 in 2024-03-07T03:18:35.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:35 smithi145 bash[79878]: cluster 2024-03-07T03:18:34.108432+0000 mon.a (mon.0) 498 : cluster 0 osdmap e454: 8 total, 8 up, 8 in 2024-03-07T03:18:35.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:35 smithi145 bash[79878]: cluster 2024-03-07T03:18:34.602638+0000 mgr.y (mgr.34107) 165 : cluster 0 pgmap v281: 105 pgs: 105 active+clean; 583 KiB data, 435 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 6.7 KiB/s wr, 66 op/s 2024-03-07T03:18:35.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:35 smithi145 bash[82329]: cluster 2024-03-07T03:18:34.108432+0000 mon.a (mon.0) 498 : cluster 0 osdmap e454: 8 total, 8 up, 8 in 2024-03-07T03:18:35.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:35 smithi145 bash[82329]: cluster 2024-03-07T03:18:34.602638+0000 mgr.y (mgr.34107) 165 : cluster 0 pgmap v281: 105 pgs: 105 active+clean; 583 KiB data, 435 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 6.7 KiB/s wr, 66 op/s 2024-03-07T03:18:35.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:35 smithi189 bash[66969]: cluster 2024-03-07T03:18:34.108432+0000 mon.a (mon.0) 498 : cluster 0 osdmap e454: 8 total, 8 up, 8 in 2024-03-07T03:18:35.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:35 smithi189 bash[66969]: cluster 2024-03-07T03:18:34.602638+0000 mgr.y (mgr.34107) 165 : cluster 0 pgmap v281: 105 pgs: 105 active+clean; 583 KiB data, 435 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 6.7 KiB/s wr, 66 op/s 2024-03-07T03:18:36.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:36 smithi145 bash[79878]: cluster 2024-03-07T03:18:35.119304+0000 mon.a (mon.0) 499 : cluster 0 osdmap e455: 8 total, 8 up, 8 in 2024-03-07T03:18:36.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:36 smithi145 bash[82329]: cluster 2024-03-07T03:18:35.119304+0000 mon.a (mon.0) 499 : cluster 0 osdmap e455: 8 total, 8 up, 8 in 2024-03-07T03:18:36.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:36 smithi189 bash[66969]: cluster 2024-03-07T03:18:35.119304+0000 mon.a (mon.0) 499 : cluster 0 osdmap e455: 8 total, 8 up, 8 in 2024-03-07T03:18:37.377 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:18:37.401 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:37 smithi145 bash[79878]: cluster 2024-03-07T03:18:36.127647+0000 mon.a (mon.0) 500 : cluster 0 osdmap e456: 8 total, 8 up, 8 in 2024-03-07T03:18:37.401 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:37 smithi145 bash[79878]: cluster 2024-03-07T03:18:36.603367+0000 mgr.y (mgr.34107) 166 : cluster 0 pgmap v284: 105 pgs: 105 active+clean; 583 KiB data, 435 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 6.7 KiB/s wr, 66 op/s 2024-03-07T03:18:37.401 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:37 smithi145 bash[82329]: cluster 2024-03-07T03:18:36.127647+0000 mon.a (mon.0) 500 : cluster 0 osdmap e456: 8 total, 8 up, 8 in 2024-03-07T03:18:37.401 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:37 smithi145 bash[82329]: cluster 2024-03-07T03:18:36.603367+0000 mgr.y (mgr.34107) 166 : cluster 0 pgmap v284: 105 pgs: 105 active+clean; 583 KiB data, 435 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 6.7 KiB/s wr, 66 op/s 2024-03-07T03:18:37.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:37 smithi189 bash[66969]: cluster 2024-03-07T03:18:36.127647+0000 mon.a (mon.0) 500 : cluster 0 osdmap e456: 8 total, 8 up, 8 in 2024-03-07T03:18:37.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:37 smithi189 bash[66969]: cluster 2024-03-07T03:18:36.603367+0000 mgr.y (mgr.34107) 166 : cluster 0 pgmap v284: 105 pgs: 105 active+clean; 583 KiB data, 435 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 6.7 KiB/s wr, 66 op/s 2024-03-07T03:18:38.521 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:38 smithi145 bash[79878]: cluster 2024-03-07T03:18:37.139581+0000 mon.a (mon.0) 501 : cluster 0 osdmap e457: 8 total, 8 up, 8 in 2024-03-07T03:18:38.521 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:38 smithi145 bash[79878]: cluster 2024-03-07T03:18:37.299779+0000 mon.a (mon.0) 502 : cluster 0 osdmap e458: 8 total, 8 up, 8 in 2024-03-07T03:18:38.521 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:38 smithi145 bash[82329]: cluster 2024-03-07T03:18:37.139581+0000 mon.a (mon.0) 501 : cluster 0 osdmap e457: 8 total, 8 up, 8 in 2024-03-07T03:18:38.521 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:38 smithi145 bash[82329]: cluster 2024-03-07T03:18:37.299779+0000 mon.a (mon.0) 502 : cluster 0 osdmap e458: 8 total, 8 up, 8 in 2024-03-07T03:18:38.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:38 smithi189 bash[66969]: cluster 2024-03-07T03:18:37.139581+0000 mon.a (mon.0) 501 : cluster 0 osdmap e457: 8 total, 8 up, 8 in 2024-03-07T03:18:38.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:38 smithi189 bash[66969]: cluster 2024-03-07T03:18:37.299779+0000 mon.a (mon.0) 502 : cluster 0 osdmap e458: 8 total, 8 up, 8 in 2024-03-07T03:18:39.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: cluster 2024-03-07T03:18:38.300046+0000 mon.a (mon.0) 503 : cluster 0 osdmap e459: 8 total, 8 up, 8 in 2024-03-07T03:18:39.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:38.385787+0000 mon.a (mon.0) 504 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:38.397476+0000 mon.a (mon.0) 505 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: cluster 2024-03-07T03:18:38.604130+0000 mgr.y (mgr.34107) 167 : cluster 0 pgmap v288: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 8.9 KiB/s wr, 131 op/s 2024-03-07T03:18:39.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.086302+0000 mon.a (mon.0) 506 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:39.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.088225+0000 mon.a (mon.0) 507 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:18:39.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.098870+0000 mon.a (mon.0) 508 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.184059+0000 mon.a (mon.0) 509 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:39.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.188969+0000 mon.a (mon.0) 510 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:18:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.193227+0000 mon.a (mon.0) 511 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:18:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.203354+0000 mon.a (mon.0) 512 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.207976+0000 mon.a (mon.0) 513 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-07T03:18:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.216035+0000 mon.a (mon.0) 514 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-07T03:18:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.220506+0000 mon.a (mon.0) 515 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-07T03:18:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.227540+0000 mon.a (mon.0) 516 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-07T03:18:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.231807+0000 mon.a (mon.0) 517 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-07T03:18:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.238784+0000 mon.a (mon.0) 518 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-07T03:18:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.244263+0000 mon.a (mon.0) 519 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:18:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.254152+0000 mon.a (mon.0) 520 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[79878]: audit 2024-03-07T03:18:39.261220+0000 mon.a (mon.0) 521 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-07T03:18:39.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: cluster 2024-03-07T03:18:38.300046+0000 mon.a (mon.0) 503 : cluster 0 osdmap e459: 8 total, 8 up, 8 in 2024-03-07T03:18:39.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:38.385787+0000 mon.a (mon.0) 504 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:38.397476+0000 mon.a (mon.0) 505 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: cluster 2024-03-07T03:18:38.604130+0000 mgr.y (mgr.34107) 167 : cluster 0 pgmap v288: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 8.9 KiB/s wr, 131 op/s 2024-03-07T03:18:39.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.086302+0000 mon.a (mon.0) 506 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:39.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.088225+0000 mon.a (mon.0) 507 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:18:39.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.098870+0000 mon.a (mon.0) 508 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.184059+0000 mon.a (mon.0) 509 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:39.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.188969+0000 mon.a (mon.0) 510 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:18:39.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.193227+0000 mon.a (mon.0) 511 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:18:39.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.203354+0000 mon.a (mon.0) 512 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.207976+0000 mon.a (mon.0) 513 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-07T03:18:39.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.216035+0000 mon.a (mon.0) 514 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-07T03:18:39.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.220506+0000 mon.a (mon.0) 515 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-07T03:18:39.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.227540+0000 mon.a (mon.0) 516 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-07T03:18:39.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.231807+0000 mon.a (mon.0) 517 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-07T03:18:39.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.238784+0000 mon.a (mon.0) 518 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-07T03:18:39.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.244263+0000 mon.a (mon.0) 519 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:18:39.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.254152+0000 mon.a (mon.0) 520 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:39 smithi145 bash[82329]: audit 2024-03-07T03:18:39.261220+0000 mon.a (mon.0) 521 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-07T03:18:39.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: cluster 2024-03-07T03:18:38.300046+0000 mon.a (mon.0) 503 : cluster 0 osdmap e459: 8 total, 8 up, 8 in 2024-03-07T03:18:39.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:38.385787+0000 mon.a (mon.0) 504 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:38.397476+0000 mon.a (mon.0) 505 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: cluster 2024-03-07T03:18:38.604130+0000 mgr.y (mgr.34107) 167 : cluster 0 pgmap v288: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 8.9 KiB/s wr, 131 op/s 2024-03-07T03:18:39.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.086302+0000 mon.a (mon.0) 506 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:39.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.088225+0000 mon.a (mon.0) 507 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:18:39.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.098870+0000 mon.a (mon.0) 508 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.184059+0000 mon.a (mon.0) 509 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:39.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.188969+0000 mon.a (mon.0) 510 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:18:39.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.193227+0000 mon.a (mon.0) 511 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:18:39.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.203354+0000 mon.a (mon.0) 512 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.207976+0000 mon.a (mon.0) 513 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-07T03:18:39.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.216035+0000 mon.a (mon.0) 514 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-07T03:18:39.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.220506+0000 mon.a (mon.0) 515 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-07T03:18:39.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.227540+0000 mon.a (mon.0) 516 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-07T03:18:39.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.231807+0000 mon.a (mon.0) 517 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-07T03:18:39.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.238784+0000 mon.a (mon.0) 518 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-07T03:18:39.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.244263+0000 mon.a (mon.0) 519 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:18:39.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.254152+0000 mon.a (mon.0) 520 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:39.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:39 smithi189 bash[66969]: audit 2024-03-07T03:18:39.261220+0000 mon.a (mon.0) 521 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-07T03:18:40.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:40 smithi189 bash[66969]: cephadm 2024-03-07T03:18:39.194920+0000 mgr.y (mgr.34107) 168 : cephadm 1 Upgrade: Setting container_image for all mon 2024-03-07T03:18:40.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:40 smithi189 bash[66969]: cephadm 2024-03-07T03:18:39.246178+0000 mgr.y (mgr.34107) 169 : cephadm 1 Upgrade: Setting container_image for all crash 2024-03-07T03:18:40.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:40 smithi189 bash[66969]: audit 2024-03-07T03:18:39.261608+0000 mgr.y (mgr.34107) 170 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-07T03:18:40.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:40 smithi189 bash[66969]: cephadm 2024-03-07T03:18:39.263891+0000 mgr.y (mgr.34107) 171 : cephadm 1 Upgrade: osd.0 is safe to restart 2024-03-07T03:18:40.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:40 smithi189 bash[66969]: cluster 2024-03-07T03:18:39.305644+0000 mon.a (mon.0) 522 : cluster 0 osdmap e460: 8 total, 8 up, 8 in 2024-03-07T03:18:40.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:40 smithi145 bash[79878]: cephadm 2024-03-07T03:18:39.194920+0000 mgr.y (mgr.34107) 168 : cephadm 1 Upgrade: Setting container_image for all mon 2024-03-07T03:18:40.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:40 smithi145 bash[79878]: cephadm 2024-03-07T03:18:39.246178+0000 mgr.y (mgr.34107) 169 : cephadm 1 Upgrade: Setting container_image for all crash 2024-03-07T03:18:40.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:40 smithi145 bash[79878]: audit 2024-03-07T03:18:39.261608+0000 mgr.y (mgr.34107) 170 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-07T03:18:40.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:40 smithi145 bash[79878]: cephadm 2024-03-07T03:18:39.263891+0000 mgr.y (mgr.34107) 171 : cephadm 1 Upgrade: osd.0 is safe to restart 2024-03-07T03:18:40.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:40 smithi145 bash[79878]: cluster 2024-03-07T03:18:39.305644+0000 mon.a (mon.0) 522 : cluster 0 osdmap e460: 8 total, 8 up, 8 in 2024-03-07T03:18:40.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:40 smithi145 bash[82329]: cephadm 2024-03-07T03:18:39.194920+0000 mgr.y (mgr.34107) 168 : cephadm 1 Upgrade: Setting container_image for all mon 2024-03-07T03:18:40.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:40 smithi145 bash[82329]: cephadm 2024-03-07T03:18:39.246178+0000 mgr.y (mgr.34107) 169 : cephadm 1 Upgrade: Setting container_image for all crash 2024-03-07T03:18:40.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:40 smithi145 bash[82329]: audit 2024-03-07T03:18:39.261608+0000 mgr.y (mgr.34107) 170 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-07T03:18:40.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:40 smithi145 bash[82329]: cephadm 2024-03-07T03:18:39.263891+0000 mgr.y (mgr.34107) 171 : cephadm 1 Upgrade: osd.0 is safe to restart 2024-03-07T03:18:40.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:40 smithi145 bash[82329]: cluster 2024-03-07T03:18:39.305644+0000 mon.a (mon.0) 522 : cluster 0 osdmap e460: 8 total, 8 up, 8 in 2024-03-07T03:18:41.067 INFO:teuthology.orchestra.run.smithi145.stdout:wait for all mons to upgrade 2024-03-07T03:18:41.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:41 smithi145 bash[82329]: cluster 2024-03-07T03:18:40.321829+0000 mon.a (mon.0) 523 : cluster 0 osdmap e461: 8 total, 8 up, 8 in 2024-03-07T03:18:41.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:41 smithi145 bash[82329]: cluster 2024-03-07T03:18:40.604816+0000 mgr.y (mgr.34107) 172 : cluster 0 pgmap v291: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 8.9 KiB/s wr, 132 op/s 2024-03-07T03:18:41.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:41 smithi145 bash[79878]: cluster 2024-03-07T03:18:40.321829+0000 mon.a (mon.0) 523 : cluster 0 osdmap e461: 8 total, 8 up, 8 in 2024-03-07T03:18:41.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:41 smithi145 bash[79878]: cluster 2024-03-07T03:18:40.604816+0000 mgr.y (mgr.34107) 172 : cluster 0 pgmap v291: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 8.9 KiB/s wr, 132 op/s 2024-03-07T03:18:41.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:41 smithi189 bash[66969]: cluster 2024-03-07T03:18:40.321829+0000 mon.a (mon.0) 523 : cluster 0 osdmap e461: 8 total, 8 up, 8 in 2024-03-07T03:18:41.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:41 smithi189 bash[66969]: cluster 2024-03-07T03:18:40.604816+0000 mgr.y (mgr.34107) 172 : cluster 0 pgmap v291: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 8.9 KiB/s wr, 132 op/s 2024-03-07T03:18:42.463 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:42 smithi145 bash[79878]: cluster 2024-03-07T03:18:41.336338+0000 mon.a (mon.0) 524 : cluster 0 osdmap e462: 8 total, 8 up, 8 in 2024-03-07T03:18:42.463 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:42 smithi145 bash[79878]: cluster 2024-03-07T03:18:42.300159+0000 mon.a (mon.0) 525 : cluster 0 osdmap e463: 8 total, 8 up, 8 in 2024-03-07T03:18:42.463 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:42 smithi145 bash[82329]: cluster 2024-03-07T03:18:41.336338+0000 mon.a (mon.0) 524 : cluster 0 osdmap e462: 8 total, 8 up, 8 in 2024-03-07T03:18:42.463 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:42 smithi145 bash[82329]: cluster 2024-03-07T03:18:42.300159+0000 mon.a (mon.0) 525 : cluster 0 osdmap e463: 8 total, 8 up, 8 in 2024-03-07T03:18:42.464 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-03-07T03:18:42.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:42 smithi189 bash[66969]: cluster 2024-03-07T03:18:41.336338+0000 mon.a (mon.0) 524 : cluster 0 osdmap e462: 8 total, 8 up, 8 in 2024-03-07T03:18:42.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:42 smithi189 bash[66969]: cluster 2024-03-07T03:18:42.300159+0000 mon.a (mon.0) 525 : cluster 0 osdmap e463: 8 total, 8 up, 8 in 2024-03-07T03:18:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:18:42 smithi189 bash[59814]: ts=2024-03-07T03:18:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:43.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:18:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:18:42] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.43.0" 2024-03-07T03:18:43.522 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.Snapshot (15472 ms) 2024-03-07T03:18:43.522 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.SnapshotRemoveOnDisable 2024-03-07T03:18:44.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[79878]: cephadm 2024-03-07T03:18:42.585791+0000 mgr.y (mgr.34107) 173 : cephadm 1 Upgrade: Updating osd.0 2024-03-07T03:18:44.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[79878]: audit 2024-03-07T03:18:42.592474+0000 mon.a (mon.0) 526 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:44.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[79878]: audit 2024-03-07T03:18:42.596952+0000 mon.a (mon.0) 527 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:18:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[79878]: audit 2024-03-07T03:18:42.598396+0000 mon.a (mon.0) 528 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[79878]: cephadm 2024-03-07T03:18:42.602146+0000 mgr.y (mgr.34107) 174 : cephadm 1 Deploying daemon osd.0 on smithi145 2024-03-07T03:18:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[79878]: cluster 2024-03-07T03:18:42.605373+0000 mgr.y (mgr.34107) 175 : cluster 0 pgmap v294: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:18:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[79878]: cluster 2024-03-07T03:18:43.344027+0000 mon.a (mon.0) 529 : cluster 0 osdmap e464: 8 total, 8 up, 8 in 2024-03-07T03:18:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[79878]: audit 2024-03-07T03:18:43.517678+0000 mon.a (mon.0) 530 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/33dc919d-7c38-4b9b-82c4-15ddac22908f"}]: dispatch 2024-03-07T03:18:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[79878]: audit 2024-03-07T03:18:43.524245+0000 mon.a (mon.0) 531 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/33dc919d-7c38-4b9b-82c4-15ddac22908f"}]': finished 2024-03-07T03:18:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[82329]: cephadm 2024-03-07T03:18:42.585791+0000 mgr.y (mgr.34107) 173 : cephadm 1 Upgrade: Updating osd.0 2024-03-07T03:18:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[82329]: audit 2024-03-07T03:18:42.592474+0000 mon.a (mon.0) 526 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[82329]: audit 2024-03-07T03:18:42.596952+0000 mon.a (mon.0) 527 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:18:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[82329]: audit 2024-03-07T03:18:42.598396+0000 mon.a (mon.0) 528 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[82329]: cephadm 2024-03-07T03:18:42.602146+0000 mgr.y (mgr.34107) 174 : cephadm 1 Deploying daemon osd.0 on smithi145 2024-03-07T03:18:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[82329]: cluster 2024-03-07T03:18:42.605373+0000 mgr.y (mgr.34107) 175 : cluster 0 pgmap v294: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:18:44.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[82329]: cluster 2024-03-07T03:18:43.344027+0000 mon.a (mon.0) 529 : cluster 0 osdmap e464: 8 total, 8 up, 8 in 2024-03-07T03:18:44.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[82329]: audit 2024-03-07T03:18:43.517678+0000 mon.a (mon.0) 530 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/33dc919d-7c38-4b9b-82c4-15ddac22908f"}]: dispatch 2024-03-07T03:18:44.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:43 smithi145 bash[82329]: audit 2024-03-07T03:18:43.524245+0000 mon.a (mon.0) 531 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/33dc919d-7c38-4b9b-82c4-15ddac22908f"}]': finished 2024-03-07T03:18:44.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:43 smithi189 bash[66969]: cephadm 2024-03-07T03:18:42.585791+0000 mgr.y (mgr.34107) 173 : cephadm 1 Upgrade: Updating osd.0 2024-03-07T03:18:44.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:43 smithi189 bash[66969]: audit 2024-03-07T03:18:42.592474+0000 mon.a (mon.0) 526 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:44.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:43 smithi189 bash[66969]: audit 2024-03-07T03:18:42.596952+0000 mon.a (mon.0) 527 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:18:44.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:43 smithi189 bash[66969]: audit 2024-03-07T03:18:42.598396+0000 mon.a (mon.0) 528 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:44.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:43 smithi189 bash[66969]: cephadm 2024-03-07T03:18:42.602146+0000 mgr.y (mgr.34107) 174 : cephadm 1 Deploying daemon osd.0 on smithi145 2024-03-07T03:18:44.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:43 smithi189 bash[66969]: cluster 2024-03-07T03:18:42.605373+0000 mgr.y (mgr.34107) 175 : cluster 0 pgmap v294: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:18:44.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:43 smithi189 bash[66969]: cluster 2024-03-07T03:18:43.344027+0000 mon.a (mon.0) 529 : cluster 0 osdmap e464: 8 total, 8 up, 8 in 2024-03-07T03:18:44.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:43 smithi189 bash[66969]: audit 2024-03-07T03:18:43.517678+0000 mon.a (mon.0) 530 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/33dc919d-7c38-4b9b-82c4-15ddac22908f"}]: dispatch 2024-03-07T03:18:44.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:43 smithi189 bash[66969]: audit 2024-03-07T03:18:43.524245+0000 mon.a (mon.0) 531 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/33dc919d-7c38-4b9b-82c4-15ddac22908f"}]': finished 2024-03-07T03:18:46.021 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:45 smithi145 bash[79878]: cluster 2024-03-07T03:18:44.600209+0000 mon.a (mon.0) 532 : cluster 0 osdmap e465: 8 total, 8 up, 8 in 2024-03-07T03:18:46.021 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:45 smithi145 bash[79878]: cluster 2024-03-07T03:18:44.606955+0000 mgr.y (mgr.34107) 176 : cluster 0 pgmap v297: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 7.2 KiB/s wr, 102 op/s 2024-03-07T03:18:46.021 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:45 smithi145 bash[82329]: cluster 2024-03-07T03:18:44.600209+0000 mon.a (mon.0) 532 : cluster 0 osdmap e465: 8 total, 8 up, 8 in 2024-03-07T03:18:46.021 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:45 smithi145 bash[82329]: cluster 2024-03-07T03:18:44.606955+0000 mgr.y (mgr.34107) 176 : cluster 0 pgmap v297: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 7.2 KiB/s wr, 102 op/s 2024-03-07T03:18:46.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:45 smithi189 bash[66969]: cluster 2024-03-07T03:18:44.600209+0000 mon.a (mon.0) 532 : cluster 0 osdmap e465: 8 total, 8 up, 8 in 2024-03-07T03:18:46.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:45 smithi189 bash[66969]: cluster 2024-03-07T03:18:44.606955+0000 mgr.y (mgr.34107) 176 : cluster 0 pgmap v297: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 7.2 KiB/s wr, 102 op/s 2024-03-07T03:18:47.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:46 smithi145 bash[79878]: cluster 2024-03-07T03:18:45.604907+0000 mon.a (mon.0) 533 : cluster 0 osdmap e466: 8 total, 8 up, 8 in 2024-03-07T03:18:47.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:46 smithi145 bash[82329]: cluster 2024-03-07T03:18:45.604907+0000 mon.a (mon.0) 533 : cluster 0 osdmap e466: 8 total, 8 up, 8 in 2024-03-07T03:18:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:46 smithi189 bash[66969]: cluster 2024-03-07T03:18:45.604907+0000 mon.a (mon.0) 533 : cluster 0 osdmap e466: 8 total, 8 up, 8 in 2024-03-07T03:18:47.448 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.SnapshotRemoveOnDisable (3926 ms) 2024-03-07T03:18:47.448 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.SnapshotUnlinkPeer 2024-03-07T03:18:47.692 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:18:47 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:47.692 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:47 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:47.693 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:18:47 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:47.693 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:18:47 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:47.694 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:47 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:47.694 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:47 smithi145 bash[79878]: cluster 2024-03-07T03:18:46.607471+0000 mgr.y (mgr.34107) 177 : cluster 0 pgmap v299: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 6.7 KiB/s wr, 95 op/s 2024-03-07T03:18:47.694 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:47 smithi145 bash[79878]: cluster 2024-03-07T03:18:46.614612+0000 mon.a (mon.0) 534 : cluster 0 osdmap e467: 8 total, 8 up, 8 in 2024-03-07T03:18:47.694 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:47 smithi145 bash[79878]: cluster 2024-03-07T03:18:47.299526+0000 mon.a (mon.0) 535 : cluster 0 osdmap e468: 8 total, 8 up, 8 in 2024-03-07T03:18:47.694 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:47 smithi145 bash[79878]: audit 2024-03-07T03:18:47.445848+0000 mon.a (mon.0) 536 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/23be2631-310b-49c3-8399-f2d850a27bc4"}]: dispatch 2024-03-07T03:18:47.694 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:47 smithi145 bash[79878]: audit 2024-03-07T03:18:47.449756+0000 mon.a (mon.0) 537 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/23be2631-310b-49c3-8399-f2d850a27bc4"}]': finished 2024-03-07T03:18:47.694 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:47 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:47.694 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:47 smithi145 bash[82329]: cluster 2024-03-07T03:18:46.607471+0000 mgr.y (mgr.34107) 177 : cluster 0 pgmap v299: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 6.7 KiB/s wr, 95 op/s 2024-03-07T03:18:47.694 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:47 smithi145 bash[82329]: cluster 2024-03-07T03:18:46.614612+0000 mon.a (mon.0) 534 : cluster 0 osdmap e467: 8 total, 8 up, 8 in 2024-03-07T03:18:47.694 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:47 smithi145 bash[82329]: cluster 2024-03-07T03:18:47.299526+0000 mon.a (mon.0) 535 : cluster 0 osdmap e468: 8 total, 8 up, 8 in 2024-03-07T03:18:47.695 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:47 smithi145 bash[82329]: audit 2024-03-07T03:18:47.445848+0000 mon.a (mon.0) 536 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/23be2631-310b-49c3-8399-f2d850a27bc4"}]: dispatch 2024-03-07T03:18:47.695 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:47 smithi145 bash[82329]: audit 2024-03-07T03:18:47.449756+0000 mon.a (mon.0) 537 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/23be2631-310b-49c3-8399-f2d850a27bc4"}]': finished 2024-03-07T03:18:47.695 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:18:47 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:47.695 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:18:47 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:47.695 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:18:47 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:47.697 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:18:48.059 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:47 smithi145 systemd[1]: Stopping Ceph osd.0 for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:18:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:47 smithi189 bash[66969]: cluster 2024-03-07T03:18:46.607471+0000 mgr.y (mgr.34107) 177 : cluster 0 pgmap v299: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 6.7 KiB/s wr, 95 op/s 2024-03-07T03:18:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:47 smithi189 bash[66969]: cluster 2024-03-07T03:18:46.614612+0000 mon.a (mon.0) 534 : cluster 0 osdmap e467: 8 total, 8 up, 8 in 2024-03-07T03:18:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:47 smithi189 bash[66969]: cluster 2024-03-07T03:18:47.299526+0000 mon.a (mon.0) 535 : cluster 0 osdmap e468: 8 total, 8 up, 8 in 2024-03-07T03:18:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:47 smithi189 bash[66969]: audit 2024-03-07T03:18:47.445848+0000 mon.a (mon.0) 536 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/23be2631-310b-49c3-8399-f2d850a27bc4"}]: dispatch 2024-03-07T03:18:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:47 smithi189 bash[66969]: audit 2024-03-07T03:18:47.449756+0000 mon.a (mon.0) 537 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/23be2631-310b-49c3-8399-f2d850a27bc4"}]': finished 2024-03-07T03:18:48.436 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[33943]: debug 2024-03-07T03:18:48.107+0000 7fb5d54f1700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:18:48.436 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[33943]: debug 2024-03-07T03:18:48.107+0000 7fb5d54f1700 -1 osd.0 468 *** Got signal Terminated *** 2024-03-07T03:18:48.437 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[33943]: debug 2024-03-07T03:18:48.107+0000 7fb5d54f1700 -1 osd.0 468 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:18:48.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[79878]: debug 2024-03-07T03:18:48.443+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:18:48.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[79878]: audit 2024-03-07T03:18:47.635708+0000 mon.a (mon.0) 538 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:48.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[79878]: cluster 2024-03-07T03:18:48.112676+0000 mon.a (mon.0) 539 : cluster 1 osd.0 marked itself down and dead 2024-03-07T03:18:48.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[79878]: cluster 2024-03-07T03:18:48.448286+0000 mon.a (mon.0) 540 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:18:48.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[79878]: cluster 2024-03-07T03:18:48.448350+0000 mon.a (mon.0) 541 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:18:48.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[79878]: cluster 2024-03-07T03:18:48.459768+0000 mon.a (mon.0) 542 : cluster 0 osdmap e469: 8 total, 7 up, 8 in 2024-03-07T03:18:48.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[82329]: audit 2024-03-07T03:18:47.635708+0000 mon.a (mon.0) 538 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:48.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[82329]: cluster 2024-03-07T03:18:48.112676+0000 mon.a (mon.0) 539 : cluster 1 osd.0 marked itself down and dead 2024-03-07T03:18:48.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[82329]: cluster 2024-03-07T03:18:48.448286+0000 mon.a (mon.0) 540 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:18:48.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[82329]: cluster 2024-03-07T03:18:48.448350+0000 mon.a (mon.0) 541 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:18:48.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:48 smithi145 bash[82329]: cluster 2024-03-07T03:18:48.459768+0000 mon.a (mon.0) 542 : cluster 0 osdmap e469: 8 total, 7 up, 8 in 2024-03-07T03:18:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:48 smithi189 bash[66969]: audit 2024-03-07T03:18:47.635708+0000 mon.a (mon.0) 538 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:48 smithi189 bash[66969]: cluster 2024-03-07T03:18:48.112676+0000 mon.a (mon.0) 539 : cluster 1 osd.0 marked itself down and dead 2024-03-07T03:18:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:48 smithi189 bash[66969]: cluster 2024-03-07T03:18:48.448286+0000 mon.a (mon.0) 540 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:18:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:48 smithi189 bash[66969]: cluster 2024-03-07T03:18:48.448350+0000 mon.a (mon.0) 541 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:18:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:48 smithi189 bash[66969]: cluster 2024-03-07T03:18:48.459768+0000 mon.a (mon.0) 542 : cluster 0 osdmap e469: 8 total, 7 up, 8 in 2024-03-07T03:18:49.874 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:49 smithi145 bash[79878]: cluster 2024-03-07T03:18:48.608972+0000 mgr.y (mgr.34107) 178 : cluster 0 pgmap v303: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 8.5 KiB/s wr, 109 op/s 2024-03-07T03:18:49.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:49 smithi145 bash[79878]: cluster 2024-03-07T03:18:49.450628+0000 mon.a (mon.0) 543 : cluster 3 Health check failed: Reduced data availability: 9 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-03-07T03:18:49.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:49 smithi145 bash[79878]: cluster 2024-03-07T03:18:49.461176+0000 mon.a (mon.0) 544 : cluster 0 osdmap e470: 8 total, 7 up, 8 in 2024-03-07T03:18:49.875 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:49 smithi145 bash[88377]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-0 2024-03-07T03:18:49.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:49 smithi145 bash[82329]: cluster 2024-03-07T03:18:48.608972+0000 mgr.y (mgr.34107) 178 : cluster 0 pgmap v303: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 8.5 KiB/s wr, 109 op/s 2024-03-07T03:18:49.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:49 smithi145 bash[82329]: cluster 2024-03-07T03:18:49.450628+0000 mon.a (mon.0) 543 : cluster 3 Health check failed: Reduced data availability: 9 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-03-07T03:18:49.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:49 smithi145 bash[82329]: cluster 2024-03-07T03:18:49.461176+0000 mon.a (mon.0) 544 : cluster 0 osdmap e470: 8 total, 7 up, 8 in 2024-03-07T03:18:50.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:49 smithi189 bash[66969]: cluster 2024-03-07T03:18:48.608972+0000 mgr.y (mgr.34107) 178 : cluster 0 pgmap v303: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 8.5 KiB/s wr, 109 op/s 2024-03-07T03:18:50.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:49 smithi189 bash[66969]: cluster 2024-03-07T03:18:49.450628+0000 mon.a (mon.0) 543 : cluster 3 Health check failed: Reduced data availability: 9 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-03-07T03:18:50.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:49 smithi189 bash[66969]: cluster 2024-03-07T03:18:49.461176+0000 mon.a (mon.0) 544 : cluster 0 osdmap e470: 8 total, 7 up, 8 in 2024-03-07T03:18:50.265 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:50 smithi145 bash[88504]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-0 2024-03-07T03:18:51.519 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:51 smithi145 bash[79878]: cluster 2024-03-07T03:18:50.470353+0000 mon.a (mon.0) 545 : cluster 0 osdmap e471: 8 total, 7 up, 8 in 2024-03-07T03:18:51.519 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:51 smithi145 bash[79878]: cluster 2024-03-07T03:18:50.609901+0000 mgr.y (mgr.34107) 179 : cluster 0 pgmap v306: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s rd, 8.5 KiB/s wr, 109 op/s 2024-03-07T03:18:51.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:51 smithi145 bash[82329]: cluster 2024-03-07T03:18:50.470353+0000 mon.a (mon.0) 545 : cluster 0 osdmap e471: 8 total, 7 up, 8 in 2024-03-07T03:18:51.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:51 smithi145 bash[82329]: cluster 2024-03-07T03:18:50.609901+0000 mgr.y (mgr.34107) 179 : cluster 0 pgmap v306: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s rd, 8.5 KiB/s wr, 109 op/s 2024-03-07T03:18:51.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:51 smithi189 bash[66969]: cluster 2024-03-07T03:18:50.470353+0000 mon.a (mon.0) 545 : cluster 0 osdmap e471: 8 total, 7 up, 8 in 2024-03-07T03:18:51.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:51 smithi189 bash[66969]: cluster 2024-03-07T03:18:50.609901+0000 mgr.y (mgr.34107) 179 : cluster 0 pgmap v306: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s rd, 8.5 KiB/s wr, 109 op/s 2024-03-07T03:18:52.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:52 smithi189 bash[66969]: cluster 2024-03-07T03:18:51.480971+0000 mon.a (mon.0) 546 : cluster 0 osdmap e472: 8 total, 7 up, 8 in 2024-03-07T03:18:52.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:52 smithi189 bash[66969]: cluster 2024-03-07T03:18:52.303167+0000 mon.a (mon.0) 547 : cluster 0 osdmap e473: 8 total, 7 up, 8 in 2024-03-07T03:18:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:18:52 smithi189 bash[59814]: ts=2024-03-07T03:18:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:52.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:52 smithi145 bash[79878]: cluster 2024-03-07T03:18:51.480971+0000 mon.a (mon.0) 546 : cluster 0 osdmap e472: 8 total, 7 up, 8 in 2024-03-07T03:18:52.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:52 smithi145 bash[79878]: cluster 2024-03-07T03:18:52.303167+0000 mon.a (mon.0) 547 : cluster 0 osdmap e473: 8 total, 7 up, 8 in 2024-03-07T03:18:52.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:52 smithi145 bash[82329]: cluster 2024-03-07T03:18:51.480971+0000 mon.a (mon.0) 546 : cluster 0 osdmap e472: 8 total, 7 up, 8 in 2024-03-07T03:18:52.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:52 smithi145 bash[82329]: cluster 2024-03-07T03:18:52.303167+0000 mon.a (mon.0) 547 : cluster 0 osdmap e473: 8 total, 7 up, 8 in 2024-03-07T03:18:52.968 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 3 2024-03-07T03:18:53.003 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:18:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:18:52] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.43.0" 2024-03-07T03:18:53.645 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:18:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:53.645 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:18:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:53.646 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:18:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:53.646 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:18:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:53.646 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:53 smithi145 bash[79878]: cluster 2024-03-07T03:18:52.610386+0000 mgr.y (mgr.34107) 180 : cluster 0 pgmap v309: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:18:53.647 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:53 smithi145 bash[79878]: audit 2024-03-07T03:18:52.967227+0000 mon.a (mon.0) 548 : audit 0 from='client.? 172.21.15.145:0/2341081318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:18:53.647 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:53 smithi145 bash[79878]: cluster 2024-03-07T03:18:53.309087+0000 mon.a (mon.0) 549 : cluster 0 osdmap e474: 8 total, 7 up, 8 in 2024-03-07T03:18:53.647 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:53 smithi145 bash[79878]: audit 2024-03-07T03:18:53.354579+0000 mon.a (mon.0) 550 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b7c7d69d-a91f-4464-86e7-a4c885d617ab"}]: dispatch 2024-03-07T03:18:53.647 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:53 smithi145 bash[79878]: audit 2024-03-07T03:18:53.362687+0000 mon.a (mon.0) 551 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b7c7d69d-a91f-4464-86e7-a4c885d617ab"}]': finished 2024-03-07T03:18:53.647 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:53.647 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:53 smithi145 bash[82329]: cluster 2024-03-07T03:18:52.610386+0000 mgr.y (mgr.34107) 180 : cluster 0 pgmap v309: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:18:53.647 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:53 smithi145 bash[82329]: audit 2024-03-07T03:18:52.967227+0000 mon.a (mon.0) 548 : audit 0 from='client.? 172.21.15.145:0/2341081318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:18:53.647 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:53 smithi145 bash[82329]: cluster 2024-03-07T03:18:53.309087+0000 mon.a (mon.0) 549 : cluster 0 osdmap e474: 8 total, 7 up, 8 in 2024-03-07T03:18:53.647 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:53 smithi145 bash[82329]: audit 2024-03-07T03:18:53.354579+0000 mon.a (mon.0) 550 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b7c7d69d-a91f-4464-86e7-a4c885d617ab"}]: dispatch 2024-03-07T03:18:53.648 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:53 smithi145 bash[82329]: audit 2024-03-07T03:18:53.362687+0000 mon.a (mon.0) 551 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b7c7d69d-a91f-4464-86e7-a4c885d617ab"}]': finished 2024-03-07T03:18:53.648 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:53.648 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:53 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.0.service: Deactivated successfully. 2024-03-07T03:18:53.648 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:53 smithi145 systemd[1]: Stopped Ceph osd.0 for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:18:53.648 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:53.648 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:18:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:53.649 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:18:53 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:53.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:53 smithi189 bash[66969]: cluster 2024-03-07T03:18:52.610386+0000 mgr.y (mgr.34107) 180 : cluster 0 pgmap v309: 105 pgs: 42 peering, 63 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:18:53.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:53 smithi189 bash[66969]: audit 2024-03-07T03:18:52.967227+0000 mon.a (mon.0) 548 : audit 0 from='client.? 172.21.15.145:0/2341081318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:18:53.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:53 smithi189 bash[66969]: cluster 2024-03-07T03:18:53.309087+0000 mon.a (mon.0) 549 : cluster 0 osdmap e474: 8 total, 7 up, 8 in 2024-03-07T03:18:53.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:53 smithi189 bash[66969]: audit 2024-03-07T03:18:53.354579+0000 mon.a (mon.0) 550 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b7c7d69d-a91f-4464-86e7-a4c885d617ab"}]: dispatch 2024-03-07T03:18:53.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:53 smithi189 bash[66969]: audit 2024-03-07T03:18:53.362687+0000 mon.a (mon.0) 551 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b7c7d69d-a91f-4464-86e7-a4c885d617ab"}]': finished 2024-03-07T03:18:54.059 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:53 smithi145 systemd[1]: Started Ceph osd.0 for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:18:54.443 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch ps' 2024-03-07T03:18:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:54 smithi145 bash[82329]: audit 2024-03-07T03:18:53.699778+0000 mon.a (mon.0) 552 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:54 smithi145 bash[82329]: audit 2024-03-07T03:18:53.708104+0000 mon.a (mon.0) 553 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:54 smithi145 bash[82329]: audit 2024-03-07T03:18:53.743836+0000 mon.a (mon.0) 554 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:54.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:54 smithi145 bash[82329]: cluster 2024-03-07T03:18:54.480666+0000 mon.a (mon.0) 555 : cluster 0 osdmap e475: 8 total, 7 up, 8 in 2024-03-07T03:18:54.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:54 smithi145 bash[79878]: audit 2024-03-07T03:18:53.699778+0000 mon.a (mon.0) 552 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:54.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:54 smithi145 bash[79878]: audit 2024-03-07T03:18:53.708104+0000 mon.a (mon.0) 553 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:54.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:54 smithi145 bash[79878]: audit 2024-03-07T03:18:53.743836+0000 mon.a (mon.0) 554 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:54.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:54 smithi145 bash[79878]: cluster 2024-03-07T03:18:54.480666+0000 mon.a (mon.0) 555 : cluster 0 osdmap e475: 8 total, 7 up, 8 in 2024-03-07T03:18:55.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:54 smithi189 bash[66969]: audit 2024-03-07T03:18:53.699778+0000 mon.a (mon.0) 552 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:55.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:54 smithi189 bash[66969]: audit 2024-03-07T03:18:53.708104+0000 mon.a (mon.0) 553 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:18:55.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:54 smithi189 bash[66969]: audit 2024-03-07T03:18:53.743836+0000 mon.a (mon.0) 554 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:55.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:54 smithi189 bash[66969]: cluster 2024-03-07T03:18:54.480666+0000 mon.a (mon.0) 555 : cluster 0 osdmap e475: 8 total, 7 up, 8 in 2024-03-07T03:18:56.036 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[79878]: cluster 2024-03-07T03:18:54.610862+0000 mgr.y (mgr.34107) 181 : cluster 0 pgmap v312: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 161 KiB/s rd, 11 KiB/s wr, 210 op/s; 53/345 objects degraded (15.362%) 2024-03-07T03:18:56.036 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[79878]: cluster 2024-03-07T03:18:55.477881+0000 mon.a (mon.0) 556 : cluster 3 Health check failed: Degraded data redundancy: 53/345 objects degraded (15.362%), 22 pgs degraded (PG_DEGRADED) 2024-03-07T03:18:56.036 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[79878]: cluster 2024-03-07T03:18:55.477932+0000 mon.a (mon.0) 557 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2024-03-07T03:18:56.036 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[79878]: cluster 2024-03-07T03:18:55.489731+0000 mon.a (mon.0) 558 : cluster 0 osdmap e476: 8 total, 7 up, 8 in 2024-03-07T03:18:56.036 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[82329]: cluster 2024-03-07T03:18:54.610862+0000 mgr.y (mgr.34107) 181 : cluster 0 pgmap v312: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 161 KiB/s rd, 11 KiB/s wr, 210 op/s; 53/345 objects degraded (15.362%) 2024-03-07T03:18:56.037 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[82329]: cluster 2024-03-07T03:18:55.477881+0000 mon.a (mon.0) 556 : cluster 3 Health check failed: Degraded data redundancy: 53/345 objects degraded (15.362%), 22 pgs degraded (PG_DEGRADED) 2024-03-07T03:18:56.037 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[82329]: cluster 2024-03-07T03:18:55.477932+0000 mon.a (mon.0) 557 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2024-03-07T03:18:56.037 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[82329]: cluster 2024-03-07T03:18:55.489731+0000 mon.a (mon.0) 558 : cluster 0 osdmap e476: 8 total, 7 up, 8 in 2024-03-07T03:18:56.037 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[88753]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-07T03:18:56.037 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[88753]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-07T03:18:56.037 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[88753]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-07T03:18:56.037 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[88753]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-07T03:18:56.037 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[88753]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-07T03:18:56.037 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[88753]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-07T03:18:56.037 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:55 smithi145 bash[88753]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-07T03:18:56.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:55 smithi189 bash[66969]: cluster 2024-03-07T03:18:54.610862+0000 mgr.y (mgr.34107) 181 : cluster 0 pgmap v312: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 161 KiB/s rd, 11 KiB/s wr, 210 op/s; 53/345 objects degraded (15.362%) 2024-03-07T03:18:56.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:55 smithi189 bash[66969]: cluster 2024-03-07T03:18:55.477881+0000 mon.a (mon.0) 556 : cluster 3 Health check failed: Degraded data redundancy: 53/345 objects degraded (15.362%), 22 pgs degraded (PG_DEGRADED) 2024-03-07T03:18:56.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:55 smithi189 bash[66969]: cluster 2024-03-07T03:18:55.477932+0000 mon.a (mon.0) 557 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2024-03-07T03:18:56.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:55 smithi189 bash[66969]: cluster 2024-03-07T03:18:55.489731+0000 mon.a (mon.0) 558 : cluster 0 osdmap e476: 8 total, 7 up, 8 in 2024-03-07T03:18:57.809 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:57 smithi145 bash[88998]: debug 2024-03-07T03:18:57.735+0000 7ff9114ea700 1 -- 172.21.15.145:0/2985478991 <== mon.0 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5638733ed680 con 0x563873428c00 2024-03-07T03:18:57.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:57 smithi145 bash[79878]: cluster 2024-03-07T03:18:56.495274+0000 mon.a (mon.0) 559 : cluster 0 osdmap e477: 8 total, 7 up, 8 in 2024-03-07T03:18:57.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:57 smithi145 bash[79878]: cluster 2024-03-07T03:18:56.611849+0000 mgr.y (mgr.34107) 182 : cluster 0 pgmap v315: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 161 KiB/s rd, 11 KiB/s wr, 210 op/s; 53/345 objects degraded (15.362%) 2024-03-07T03:18:57.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:57 smithi145 bash[82329]: cluster 2024-03-07T03:18:56.495274+0000 mon.a (mon.0) 559 : cluster 0 osdmap e477: 8 total, 7 up, 8 in 2024-03-07T03:18:57.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:57 smithi145 bash[82329]: cluster 2024-03-07T03:18:56.611849+0000 mgr.y (mgr.34107) 182 : cluster 0 pgmap v315: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 161 KiB/s rd, 11 KiB/s wr, 210 op/s; 53/345 objects degraded (15.362%) 2024-03-07T03:18:57.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:57 smithi189 bash[66969]: cluster 2024-03-07T03:18:56.495274+0000 mon.a (mon.0) 559 : cluster 0 osdmap e477: 8 total, 7 up, 8 in 2024-03-07T03:18:57.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:57 smithi189 bash[66969]: cluster 2024-03-07T03:18:56.611849+0000 mgr.y (mgr.34107) 182 : cluster 0 pgmap v315: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 161 KiB/s rd, 11 KiB/s wr, 210 op/s; 53/345 objects degraded (15.362%) 2024-03-07T03:18:58.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:58 smithi145 bash[79878]: cluster 2024-03-07T03:18:57.504421+0000 mon.a (mon.0) 560 : cluster 0 osdmap e478: 8 total, 7 up, 8 in 2024-03-07T03:18:58.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:58 smithi145 bash[82329]: cluster 2024-03-07T03:18:57.504421+0000 mon.a (mon.0) 560 : cluster 0 osdmap e478: 8 total, 7 up, 8 in 2024-03-07T03:18:58.560 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:18:58 smithi145 bash[88998]: debug 2024-03-07T03:18:58.271+0000 7ff91a6827c0 -1 Falling back to public interface 2024-03-07T03:18:58.655 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.SnapshotUnlinkPeer (11206 ms) 2024-03-07T03:18:58.655 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.SnapshotImageState 2024-03-07T03:18:58.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:58 smithi189 bash[66969]: cluster 2024-03-07T03:18:57.504421+0000 mon.a (mon.0) 560 : cluster 0 osdmap e478: 8 total, 7 up, 8 in 2024-03-07T03:18:59.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:59 smithi145 bash[79878]: cluster 2024-03-07T03:18:58.513762+0000 mon.a (mon.0) 561 : cluster 0 osdmap e479: 8 total, 7 up, 8 in 2024-03-07T03:18:59.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:59 smithi145 bash[79878]: cluster 2024-03-07T03:18:58.612850+0000 mgr.y (mgr.34107) 183 : cluster 0 pgmap v318: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 6.7 KiB/s wr, 108 op/s; 49/312 objects degraded (15.705%) 2024-03-07T03:18:59.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:59 smithi145 bash[79878]: audit 2024-03-07T03:18:58.640713+0000 mon.a (mon.0) 562 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/4e5ebedc-14b0-4be7-9275-355fff197386"}]: dispatch 2024-03-07T03:18:59.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:59 smithi145 bash[79878]: audit 2024-03-07T03:18:58.645740+0000 mon.a (mon.0) 563 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/4e5ebedc-14b0-4be7-9275-355fff197386"}]': finished 2024-03-07T03:18:59.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:59 smithi145 bash[79878]: audit 2024-03-07T03:18:58.651346+0000 mon.a (mon.0) 564 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0eef6d00-3c3f-4565-8090-7f82cdce892f"}]: dispatch 2024-03-07T03:18:59.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:18:59 smithi145 bash[79878]: audit 2024-03-07T03:18:58.654702+0000 mon.a (mon.0) 565 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0eef6d00-3c3f-4565-8090-7f82cdce892f"}]': finished 2024-03-07T03:18:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:59 smithi145 bash[82329]: cluster 2024-03-07T03:18:58.513762+0000 mon.a (mon.0) 561 : cluster 0 osdmap e479: 8 total, 7 up, 8 in 2024-03-07T03:18:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:59 smithi145 bash[82329]: cluster 2024-03-07T03:18:58.612850+0000 mgr.y (mgr.34107) 183 : cluster 0 pgmap v318: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 6.7 KiB/s wr, 108 op/s; 49/312 objects degraded (15.705%) 2024-03-07T03:18:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:59 smithi145 bash[82329]: audit 2024-03-07T03:18:58.640713+0000 mon.a (mon.0) 562 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/4e5ebedc-14b0-4be7-9275-355fff197386"}]: dispatch 2024-03-07T03:18:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:59 smithi145 bash[82329]: audit 2024-03-07T03:18:58.645740+0000 mon.a (mon.0) 563 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/4e5ebedc-14b0-4be7-9275-355fff197386"}]': finished 2024-03-07T03:18:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:59 smithi145 bash[82329]: audit 2024-03-07T03:18:58.651346+0000 mon.a (mon.0) 564 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0eef6d00-3c3f-4565-8090-7f82cdce892f"}]: dispatch 2024-03-07T03:18:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:18:59 smithi145 bash[82329]: audit 2024-03-07T03:18:58.654702+0000 mon.a (mon.0) 565 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0eef6d00-3c3f-4565-8090-7f82cdce892f"}]': finished 2024-03-07T03:18:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:59 smithi189 bash[66969]: cluster 2024-03-07T03:18:58.513762+0000 mon.a (mon.0) 561 : cluster 0 osdmap e479: 8 total, 7 up, 8 in 2024-03-07T03:18:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:59 smithi189 bash[66969]: cluster 2024-03-07T03:18:58.612850+0000 mgr.y (mgr.34107) 183 : cluster 0 pgmap v318: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 6.7 KiB/s wr, 108 op/s; 49/312 objects degraded (15.705%) 2024-03-07T03:18:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:59 smithi189 bash[66969]: audit 2024-03-07T03:18:58.640713+0000 mon.a (mon.0) 562 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/4e5ebedc-14b0-4be7-9275-355fff197386"}]: dispatch 2024-03-07T03:18:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:59 smithi189 bash[66969]: audit 2024-03-07T03:18:58.645740+0000 mon.a (mon.0) 563 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/4e5ebedc-14b0-4be7-9275-355fff197386"}]': finished 2024-03-07T03:18:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:59 smithi189 bash[66969]: audit 2024-03-07T03:18:58.651346+0000 mon.a (mon.0) 564 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0eef6d00-3c3f-4565-8090-7f82cdce892f"}]: dispatch 2024-03-07T03:18:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:18:59 smithi189 bash[66969]: audit 2024-03-07T03:18:58.654702+0000 mon.a (mon.0) 565 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0eef6d00-3c3f-4565-8090-7f82cdce892f"}]': finished 2024-03-07T03:19:00.209 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:19:01.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:00 smithi145 bash[79878]: cluster 2024-03-07T03:18:59.660815+0000 mon.a (mon.0) 566 : cluster 0 osdmap e480: 8 total, 7 up, 8 in 2024-03-07T03:19:01.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:00 smithi145 bash[82329]: cluster 2024-03-07T03:18:59.660815+0000 mon.a (mon.0) 566 : cluster 0 osdmap e480: 8 total, 7 up, 8 in 2024-03-07T03:19:01.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:00 smithi189 bash[66969]: cluster 2024-03-07T03:18:59.660815+0000 mon.a (mon.0) 566 : cluster 0 osdmap e480: 8 total, 7 up, 8 in 2024-03-07T03:19:01.652 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:19:01 smithi145 bash[88998]: debug 2024-03-07T03:19:01.419+0000 7ff91a6827c0 -1 osd.0 468 log_to_monitors true 2024-03-07T03:19:02.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:01 smithi145 bash[82329]: cluster 2024-03-07T03:19:00.613369+0000 mgr.y (mgr.34107) 184 : cluster 0 pgmap v320: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 105 op/s; 49/312 objects degraded (15.705%) 2024-03-07T03:19:02.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:01 smithi145 bash[82329]: cluster 2024-03-07T03:19:00.671689+0000 mon.a (mon.0) 567 : cluster 0 osdmap e481: 8 total, 7 up, 8 in 2024-03-07T03:19:02.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:01 smithi145 bash[82329]: audit 2024-03-07T03:19:01.131250+0000 mon.a (mon.0) 568 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:02.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:01 smithi145 bash[82329]: audit 2024-03-07T03:19:01.138041+0000 mon.a (mon.0) 569 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:02.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:01 smithi145 bash[82329]: audit 2024-03-07T03:19:01.427867+0000 mon.a (mon.0) 570 : audit 1 from='osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T03:19:02.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:01 smithi145 bash[79878]: cluster 2024-03-07T03:19:00.613369+0000 mgr.y (mgr.34107) 184 : cluster 0 pgmap v320: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 105 op/s; 49/312 objects degraded (15.705%) 2024-03-07T03:19:02.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:01 smithi145 bash[79878]: cluster 2024-03-07T03:19:00.671689+0000 mon.a (mon.0) 567 : cluster 0 osdmap e481: 8 total, 7 up, 8 in 2024-03-07T03:19:02.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:01 smithi145 bash[79878]: audit 2024-03-07T03:19:01.131250+0000 mon.a (mon.0) 568 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:02.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:01 smithi145 bash[79878]: audit 2024-03-07T03:19:01.138041+0000 mon.a (mon.0) 569 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:02.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:01 smithi145 bash[79878]: audit 2024-03-07T03:19:01.427867+0000 mon.a (mon.0) 570 : audit 1 from='osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T03:19:02.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:01 smithi189 bash[66969]: cluster 2024-03-07T03:19:00.613369+0000 mgr.y (mgr.34107) 184 : cluster 0 pgmap v320: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 105 op/s; 49/312 objects degraded (15.705%) 2024-03-07T03:19:02.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:01 smithi189 bash[66969]: cluster 2024-03-07T03:19:00.671689+0000 mon.a (mon.0) 567 : cluster 0 osdmap e481: 8 total, 7 up, 8 in 2024-03-07T03:19:02.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:01 smithi189 bash[66969]: audit 2024-03-07T03:19:01.131250+0000 mon.a (mon.0) 568 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:02.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:01 smithi189 bash[66969]: audit 2024-03-07T03:19:01.138041+0000 mon.a (mon.0) 569 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:02.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:01 smithi189 bash[66969]: audit 2024-03-07T03:19:01.427867+0000 mon.a (mon.0) 570 : audit 1 from='osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T03:19:02.559 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[88998]: debug 2024-03-07T03:19:02.179+0000 7ff907ad2700 -1 osd.0 468 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-07T03:19:02.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:19:02 smithi189 bash[59814]: ts=2024-03-07T03:19:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:03.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:19:02] "GET /metrics HTTP/1.1" 200 36290 "" "Prometheus/2.43.0" 2024-03-07T03:19:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[82329]: audit 2024-03-07T03:19:01.673941+0000 mon.a (mon.0) 571 : audit 1 from='osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-07T03:19:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[82329]: cluster 2024-03-07T03:19:01.679720+0000 mon.a (mon.0) 572 : cluster 0 osdmap e482: 8 total, 7 up, 8 in 2024-03-07T03:19:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[82329]: audit 2024-03-07T03:19:01.680084+0000 mon.a (mon.0) 573 : audit 1 from='osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:19:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[82329]: audit 2024-03-07T03:19:02.131642+0000 mon.a (mon.0) 574 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[82329]: audit 2024-03-07T03:19:02.139076+0000 mon.a (mon.0) 575 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[82329]: audit 2024-03-07T03:19:02.158217+0000 mon.a (mon.0) 576 : audit 1 from='osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411]' entity='osd.0' 2024-03-07T03:19:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[82329]: cluster 2024-03-07T03:19:02.296338+0000 mon.a (mon.0) 577 : cluster 3 Health check update: Degraded data redundancy: 49/312 objects degraded (15.705%), 22 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[82329]: audit 2024-03-07T03:19:02.643276+0000 mon.a (mon.0) 578 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[82329]: audit 2024-03-07T03:19:02.645257+0000 mon.a (mon.0) 579 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:03.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[79878]: audit 2024-03-07T03:19:01.673941+0000 mon.a (mon.0) 571 : audit 1 from='osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-07T03:19:03.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[79878]: cluster 2024-03-07T03:19:01.679720+0000 mon.a (mon.0) 572 : cluster 0 osdmap e482: 8 total, 7 up, 8 in 2024-03-07T03:19:03.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[79878]: audit 2024-03-07T03:19:01.680084+0000 mon.a (mon.0) 573 : audit 1 from='osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:19:03.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[79878]: audit 2024-03-07T03:19:02.131642+0000 mon.a (mon.0) 574 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:03.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[79878]: audit 2024-03-07T03:19:02.139076+0000 mon.a (mon.0) 575 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:03.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[79878]: audit 2024-03-07T03:19:02.158217+0000 mon.a (mon.0) 576 : audit 1 from='osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411]' entity='osd.0' 2024-03-07T03:19:03.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[79878]: cluster 2024-03-07T03:19:02.296338+0000 mon.a (mon.0) 577 : cluster 3 Health check update: Degraded data redundancy: 49/312 objects degraded (15.705%), 22 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:03.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[79878]: audit 2024-03-07T03:19:02.643276+0000 mon.a (mon.0) 578 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:03.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[79878]: audit 2024-03-07T03:19:02.645257+0000 mon.a (mon.0) 579 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:03.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:02 smithi145 bash[79878]: debug 2024-03-07T03:19:02.671+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:19:03.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:02 smithi189 bash[66969]: audit 2024-03-07T03:19:01.673941+0000 mon.a (mon.0) 571 : audit 1 from='osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-07T03:19:03.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:02 smithi189 bash[66969]: cluster 2024-03-07T03:19:01.679720+0000 mon.a (mon.0) 572 : cluster 0 osdmap e482: 8 total, 7 up, 8 in 2024-03-07T03:19:03.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:02 smithi189 bash[66969]: audit 2024-03-07T03:19:01.680084+0000 mon.a (mon.0) 573 : audit 1 from='osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:19:03.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:02 smithi189 bash[66969]: audit 2024-03-07T03:19:02.131642+0000 mon.a (mon.0) 574 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:03.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:02 smithi189 bash[66969]: audit 2024-03-07T03:19:02.139076+0000 mon.a (mon.0) 575 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:03.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:02 smithi189 bash[66969]: audit 2024-03-07T03:19:02.158217+0000 mon.a (mon.0) 576 : audit 1 from='osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411]' entity='osd.0' 2024-03-07T03:19:03.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:02 smithi189 bash[66969]: cluster 2024-03-07T03:19:02.296338+0000 mon.a (mon.0) 577 : cluster 3 Health check update: Degraded data redundancy: 49/312 objects degraded (15.705%), 22 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:02 smithi189 bash[66969]: audit 2024-03-07T03:19:02.643276+0000 mon.a (mon.0) 578 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:02 smithi189 bash[66969]: audit 2024-03-07T03:19:02.645257+0000 mon.a (mon.0) 579 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:03.800 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.SnapshotImageState (5146 ms) 2024-03-07T03:19:03.801 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.SnapshotPromoteDemote 2024-03-07T03:19:03.839 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-07T03:19:03.839 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (11m) 2s ago 12m 14.3M - 0.25.0 c8568f914cd2 cfa1e64a6007 2024-03-07T03:19:03.839 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi189 *:3000 running (12m) 25s ago 12m 79.6M - 9.4.7 954c08fa6188 89badf11bd11 2024-03-07T03:19:03.839 INFO:teuthology.orchestra.run.smithi145.stdout:mds.foo.smithi145.zmgvgw smithi145 running (10m) 2s ago 10m 15.1M - 18.2.1-718-gae75fc9d 0164bb8a7c67 4ded79fef80e 2024-03-07T03:19:03.839 INFO:teuthology.orchestra.run.smithi145.stdout:mds.foo.smithi189.zkinbe smithi189 running (10m) 25s ago 10m 18.4M - 18.2.1-718-gae75fc9d 0164bb8a7c67 6589deb5c181 2024-03-07T03:19:03.839 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi189 *:8443,9283,8765 running (4m) 25s ago 20m 437M - 19.0.0-1905-gea9a5688 fdb83767c1ab acef232ce9b9 2024-03-07T03:19:03.839 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (5m) 2s ago 23m 494M - 19.0.0-1905-gea9a5688 fdb83767c1ab 2cae6fe389e0 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (4m) 2s ago 23m 53.3M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 8a2f4984bf09 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi189 running (112s) 25s ago 21m 36.6M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab c2bb73c80f4c 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (2m) 2s ago 21m 39.9M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 1c9cf0604d03 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (12m) 2s ago 12m 8819k - 1.5.0 0da6a335fe13 895e660f39b6 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi189 *:9100 running (12m) 25s ago 12m 9020k - 1.5.0 0da6a335fe13 ab7ffb60c1e8 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (7s) 2s ago 20m - 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab b5501c15a04b 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (19m) 2s ago 19m 123M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 f21d0bf7ebc8 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (18m) 2s ago 18m 139M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 9e8c1bf873ba 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (17m) 2s ago 17m 127M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 c209b1a43303 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi189 running (16m) 25s ago 16m 128M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 8845b4f5216a 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi189 running (15m) 25s ago 15m 111M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 44e5411eb07c 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi189 running (14m) 25s ago 14m 136M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 c2aeea0466c7 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi189 running (14m) 25s ago 14m 108M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 bf101bfb6800 2024-03-07T03:19:03.840 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi189 *:9095 running (4m) 25s ago 13m 50.6M - 2.43.0 a07b618ecd1d 144e5cadeea3 2024-03-07T03:19:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:03 smithi145 bash[79878]: cluster 2024-03-07T03:19:02.614344+0000 mgr.y (mgr.34107) 185 : cluster 0 pgmap v323: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 421 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 6.6 KiB/s wr, 105 op/s; 49/312 objects degraded (15.705%) 2024-03-07T03:19:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:03 smithi145 bash[79878]: cluster 2024-03-07T03:19:02.674842+0000 mon.a (mon.0) 580 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:19:04.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:03 smithi145 bash[79878]: cluster 2024-03-07T03:19:02.674865+0000 mon.a (mon.0) 581 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:19:04.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:03 smithi145 bash[79878]: cluster 2024-03-07T03:19:02.698376+0000 mon.a (mon.0) 582 : cluster 1 osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411] boot 2024-03-07T03:19:04.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:03 smithi145 bash[79878]: cluster 2024-03-07T03:19:02.698692+0000 mon.a (mon.0) 583 : cluster 0 osdmap e483: 8 total, 8 up, 8 in 2024-03-07T03:19:04.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:03 smithi145 bash[79878]: audit 2024-03-07T03:19:02.699661+0000 mon.a (mon.0) 584 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:19:04.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:03 smithi145 bash[82329]: cluster 2024-03-07T03:19:02.614344+0000 mgr.y (mgr.34107) 185 : cluster 0 pgmap v323: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 421 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 6.6 KiB/s wr, 105 op/s; 49/312 objects degraded (15.705%) 2024-03-07T03:19:04.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:03 smithi145 bash[82329]: cluster 2024-03-07T03:19:02.674842+0000 mon.a (mon.0) 580 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:19:04.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:03 smithi145 bash[82329]: cluster 2024-03-07T03:19:02.674865+0000 mon.a (mon.0) 581 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:19:04.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:03 smithi145 bash[82329]: cluster 2024-03-07T03:19:02.698376+0000 mon.a (mon.0) 582 : cluster 1 osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411] boot 2024-03-07T03:19:04.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:03 smithi145 bash[82329]: cluster 2024-03-07T03:19:02.698692+0000 mon.a (mon.0) 583 : cluster 0 osdmap e483: 8 total, 8 up, 8 in 2024-03-07T03:19:04.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:03 smithi145 bash[82329]: audit 2024-03-07T03:19:02.699661+0000 mon.a (mon.0) 584 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:19:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:03 smithi189 bash[66969]: cluster 2024-03-07T03:19:02.614344+0000 mgr.y (mgr.34107) 185 : cluster 0 pgmap v323: 105 pgs: 22 active+undersized+degraded, 20 active+undersized, 63 active+clean; 583 KiB data, 421 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 6.6 KiB/s wr, 105 op/s; 49/312 objects degraded (15.705%) 2024-03-07T03:19:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:03 smithi189 bash[66969]: cluster 2024-03-07T03:19:02.674842+0000 mon.a (mon.0) 580 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:19:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:03 smithi189 bash[66969]: cluster 2024-03-07T03:19:02.674865+0000 mon.a (mon.0) 581 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:19:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:03 smithi189 bash[66969]: cluster 2024-03-07T03:19:02.698376+0000 mon.a (mon.0) 582 : cluster 1 osd.0 [v2:172.21.15.145:6802/3509507411,v1:172.21.15.145:6803/3509507411] boot 2024-03-07T03:19:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:03 smithi189 bash[66969]: cluster 2024-03-07T03:19:02.698692+0000 mon.a (mon.0) 583 : cluster 0 osdmap e483: 8 total, 8 up, 8 in 2024-03-07T03:19:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:03 smithi189 bash[66969]: audit 2024-03-07T03:19:02.699661+0000 mon.a (mon.0) 584 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:19:04.819 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade pause' 2024-03-07T03:19:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:04 smithi145 bash[82329]: cluster 2024-03-07T03:19:03.703186+0000 mon.a (mon.0) 585 : cluster 0 osdmap e484: 8 total, 8 up, 8 in 2024-03-07T03:19:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:04 smithi145 bash[82329]: audit 2024-03-07T03:19:03.836862+0000 mgr.y (mgr.34107) 186 : audit 0 from='client.44168 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:19:05.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:04 smithi145 bash[79878]: cluster 2024-03-07T03:19:03.703186+0000 mon.a (mon.0) 585 : cluster 0 osdmap e484: 8 total, 8 up, 8 in 2024-03-07T03:19:05.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:04 smithi145 bash[79878]: audit 2024-03-07T03:19:03.836862+0000 mgr.y (mgr.34107) 186 : audit 0 from='client.44168 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:19:05.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:04 smithi189 bash[66969]: cluster 2024-03-07T03:19:03.703186+0000 mon.a (mon.0) 585 : cluster 0 osdmap e484: 8 total, 8 up, 8 in 2024-03-07T03:19:05.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:04 smithi189 bash[66969]: audit 2024-03-07T03:19:03.836862+0000 mgr.y (mgr.34107) 186 : audit 0 from='client.44168 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:19:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:05 smithi145 bash[79878]: cluster 2024-03-07T03:19:04.614843+0000 mgr.y (mgr.34107) 187 : cluster 0 pgmap v326: 105 pgs: 24 peering, 9 active+undersized+degraded, 9 active+undersized, 63 active+clean; 583 KiB data, 426 MiB used, 715 GiB / 715 GiB avail; 141 KiB/s rd, 19 KiB/s wr, 196 op/s; 20/306 objects degraded (6.536%) 2024-03-07T03:19:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:05 smithi145 bash[79878]: cluster 2024-03-07T03:19:04.709549+0000 mon.a (mon.0) 586 : cluster 0 osdmap e485: 8 total, 8 up, 8 in 2024-03-07T03:19:06.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:05 smithi145 bash[82329]: cluster 2024-03-07T03:19:04.614843+0000 mgr.y (mgr.34107) 187 : cluster 0 pgmap v326: 105 pgs: 24 peering, 9 active+undersized+degraded, 9 active+undersized, 63 active+clean; 583 KiB data, 426 MiB used, 715 GiB / 715 GiB avail; 141 KiB/s rd, 19 KiB/s wr, 196 op/s; 20/306 objects degraded (6.536%) 2024-03-07T03:19:06.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:05 smithi145 bash[82329]: cluster 2024-03-07T03:19:04.709549+0000 mon.a (mon.0) 586 : cluster 0 osdmap e485: 8 total, 8 up, 8 in 2024-03-07T03:19:06.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:05 smithi189 bash[66969]: cluster 2024-03-07T03:19:04.614843+0000 mgr.y (mgr.34107) 187 : cluster 0 pgmap v326: 105 pgs: 24 peering, 9 active+undersized+degraded, 9 active+undersized, 63 active+clean; 583 KiB data, 426 MiB used, 715 GiB / 715 GiB avail; 141 KiB/s rd, 19 KiB/s wr, 196 op/s; 20/306 objects degraded (6.536%) 2024-03-07T03:19:06.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:05 smithi189 bash[66969]: cluster 2024-03-07T03:19:04.709549+0000 mon.a (mon.0) 586 : cluster 0 osdmap e485: 8 total, 8 up, 8 in 2024-03-07T03:19:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:06 smithi145 bash[79878]: cluster 2024-03-07T03:19:05.709173+0000 mon.a (mon.0) 587 : cluster 0 osdmap e486: 8 total, 8 up, 8 in 2024-03-07T03:19:07.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:06 smithi145 bash[82329]: cluster 2024-03-07T03:19:05.709173+0000 mon.a (mon.0) 587 : cluster 0 osdmap e486: 8 total, 8 up, 8 in 2024-03-07T03:19:07.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:06 smithi189 bash[66969]: cluster 2024-03-07T03:19:05.709173+0000 mon.a (mon.0) 587 : cluster 0 osdmap e486: 8 total, 8 up, 8 in 2024-03-07T03:19:08.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:07 smithi145 bash[82329]: cluster 2024-03-07T03:19:06.615680+0000 mgr.y (mgr.34107) 188 : cluster 0 pgmap v329: 105 pgs: 24 peering, 9 active+undersized+degraded, 9 active+undersized, 63 active+clean; 583 KiB data, 426 MiB used, 715 GiB / 715 GiB avail; 141 KiB/s rd, 19 KiB/s wr, 196 op/s; 20/306 objects degraded (6.536%) 2024-03-07T03:19:08.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:07 smithi145 bash[82329]: cluster 2024-03-07T03:19:06.731151+0000 mon.a (mon.0) 588 : cluster 0 osdmap e487: 8 total, 8 up, 8 in 2024-03-07T03:19:08.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:07 smithi145 bash[82329]: cluster 2024-03-07T03:19:07.297397+0000 mon.a (mon.0) 589 : cluster 3 Health check update: Degraded data redundancy: 20/306 objects degraded (6.536%), 9 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:08.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:07 smithi145 bash[79878]: cluster 2024-03-07T03:19:06.615680+0000 mgr.y (mgr.34107) 188 : cluster 0 pgmap v329: 105 pgs: 24 peering, 9 active+undersized+degraded, 9 active+undersized, 63 active+clean; 583 KiB data, 426 MiB used, 715 GiB / 715 GiB avail; 141 KiB/s rd, 19 KiB/s wr, 196 op/s; 20/306 objects degraded (6.536%) 2024-03-07T03:19:08.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:07 smithi145 bash[79878]: cluster 2024-03-07T03:19:06.731151+0000 mon.a (mon.0) 588 : cluster 0 osdmap e487: 8 total, 8 up, 8 in 2024-03-07T03:19:08.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:07 smithi145 bash[79878]: cluster 2024-03-07T03:19:07.297397+0000 mon.a (mon.0) 589 : cluster 3 Health check update: Degraded data redundancy: 20/306 objects degraded (6.536%), 9 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:08.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:07 smithi189 bash[66969]: cluster 2024-03-07T03:19:06.615680+0000 mgr.y (mgr.34107) 188 : cluster 0 pgmap v329: 105 pgs: 24 peering, 9 active+undersized+degraded, 9 active+undersized, 63 active+clean; 583 KiB data, 426 MiB used, 715 GiB / 715 GiB avail; 141 KiB/s rd, 19 KiB/s wr, 196 op/s; 20/306 objects degraded (6.536%) 2024-03-07T03:19:08.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:07 smithi189 bash[66969]: cluster 2024-03-07T03:19:06.731151+0000 mon.a (mon.0) 588 : cluster 0 osdmap e487: 8 total, 8 up, 8 in 2024-03-07T03:19:08.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:07 smithi189 bash[66969]: cluster 2024-03-07T03:19:07.297397+0000 mon.a (mon.0) 589 : cluster 3 Health check update: Degraded data redundancy: 20/306 objects degraded (6.536%), 9 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:09.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:08 smithi145 bash[79878]: cluster 2024-03-07T03:19:07.735270+0000 mon.a (mon.0) 590 : cluster 0 osdmap e488: 8 total, 8 up, 8 in 2024-03-07T03:19:09.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:08 smithi145 bash[82329]: cluster 2024-03-07T03:19:07.735270+0000 mon.a (mon.0) 590 : cluster 0 osdmap e488: 8 total, 8 up, 8 in 2024-03-07T03:19:09.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:08 smithi189 bash[66969]: cluster 2024-03-07T03:19:07.735270+0000 mon.a (mon.0) 590 : cluster 0 osdmap e488: 8 total, 8 up, 8 in 2024-03-07T03:19:09.641 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:19:10.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:09 smithi189 bash[66969]: cluster 2024-03-07T03:19:08.616973+0000 mgr.y (mgr.34107) 189 : cluster 0 pgmap v332: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 11 KiB/s wr, 149 op/s; 8 B/s, 0 keys/s, 8 objects/s recovering 2024-03-07T03:19:10.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:09 smithi189 bash[66969]: cluster 2024-03-07T03:19:08.737976+0000 mon.a (mon.0) 591 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/306 objects degraded (6.536%), 9 pgs degraded) 2024-03-07T03:19:10.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:09 smithi189 bash[66969]: cluster 2024-03-07T03:19:08.758412+0000 mon.a (mon.0) 592 : cluster 0 osdmap e489: 8 total, 8 up, 8 in 2024-03-07T03:19:10.086 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:09 smithi145 bash[79878]: cluster 2024-03-07T03:19:08.616973+0000 mgr.y (mgr.34107) 189 : cluster 0 pgmap v332: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 11 KiB/s wr, 149 op/s; 8 B/s, 0 keys/s, 8 objects/s recovering 2024-03-07T03:19:10.087 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:09 smithi145 bash[79878]: cluster 2024-03-07T03:19:08.737976+0000 mon.a (mon.0) 591 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/306 objects degraded (6.536%), 9 pgs degraded) 2024-03-07T03:19:10.087 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:09 smithi145 bash[79878]: cluster 2024-03-07T03:19:08.758412+0000 mon.a (mon.0) 592 : cluster 0 osdmap e489: 8 total, 8 up, 8 in 2024-03-07T03:19:10.087 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:09 smithi145 bash[82329]: cluster 2024-03-07T03:19:08.616973+0000 mgr.y (mgr.34107) 189 : cluster 0 pgmap v332: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 11 KiB/s wr, 149 op/s; 8 B/s, 0 keys/s, 8 objects/s recovering 2024-03-07T03:19:10.087 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:09 smithi145 bash[82329]: cluster 2024-03-07T03:19:08.737976+0000 mon.a (mon.0) 591 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/306 objects degraded (6.536%), 9 pgs degraded) 2024-03-07T03:19:10.087 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:09 smithi145 bash[82329]: cluster 2024-03-07T03:19:08.758412+0000 mon.a (mon.0) 592 : cluster 0 osdmap e489: 8 total, 8 up, 8 in 2024-03-07T03:19:11.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:10 smithi145 bash[82329]: cluster 2024-03-07T03:19:09.768355+0000 mon.a (mon.0) 593 : cluster 0 osdmap e490: 8 total, 8 up, 8 in 2024-03-07T03:19:11.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:10 smithi145 bash[79878]: cluster 2024-03-07T03:19:09.768355+0000 mon.a (mon.0) 593 : cluster 0 osdmap e490: 8 total, 8 up, 8 in 2024-03-07T03:19:11.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:10 smithi189 bash[66969]: cluster 2024-03-07T03:19:09.768355+0000 mon.a (mon.0) 593 : cluster 0 osdmap e490: 8 total, 8 up, 8 in 2024-03-07T03:19:11.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:11 smithi145 bash[79878]: cluster 2024-03-07T03:19:10.617694+0000 mgr.y (mgr.34107) 190 : cluster 0 pgmap v335: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 11 KiB/s wr, 149 op/s; 8 B/s, 0 keys/s, 8 objects/s recovering 2024-03-07T03:19:11.813 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:11 smithi145 bash[79878]: cluster 2024-03-07T03:19:10.776470+0000 mon.a (mon.0) 594 : cluster 0 osdmap e491: 8 total, 8 up, 8 in 2024-03-07T03:19:11.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:11 smithi145 bash[82329]: cluster 2024-03-07T03:19:10.617694+0000 mgr.y (mgr.34107) 190 : cluster 0 pgmap v335: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 11 KiB/s wr, 149 op/s; 8 B/s, 0 keys/s, 8 objects/s recovering 2024-03-07T03:19:11.813 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:11 smithi145 bash[82329]: cluster 2024-03-07T03:19:10.776470+0000 mon.a (mon.0) 594 : cluster 0 osdmap e491: 8 total, 8 up, 8 in 2024-03-07T03:19:12.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:11 smithi189 bash[66969]: cluster 2024-03-07T03:19:10.617694+0000 mgr.y (mgr.34107) 190 : cluster 0 pgmap v335: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 11 KiB/s wr, 149 op/s; 8 B/s, 0 keys/s, 8 objects/s recovering 2024-03-07T03:19:12.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:11 smithi189 bash[66969]: cluster 2024-03-07T03:19:10.776470+0000 mon.a (mon.0) 594 : cluster 0 osdmap e491: 8 total, 8 up, 8 in 2024-03-07T03:19:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:19:12 smithi189 bash[59814]: ts=2024-03-07T03:19:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:13.033 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:19:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:19:12] "GET /metrics HTTP/1.1" 200 36290 "" "Prometheus/2.43.0" 2024-03-07T03:19:13.033 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:12 smithi145 bash[79878]: cluster 2024-03-07T03:19:11.780364+0000 mon.a (mon.0) 595 : cluster 0 osdmap e492: 8 total, 8 up, 8 in 2024-03-07T03:19:13.033 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:12 smithi145 bash[79878]: cluster 2024-03-07T03:19:12.303813+0000 mon.a (mon.0) 596 : cluster 0 osdmap e493: 8 total, 8 up, 8 in 2024-03-07T03:19:13.034 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:12 smithi145 bash[82329]: cluster 2024-03-07T03:19:11.780364+0000 mon.a (mon.0) 595 : cluster 0 osdmap e492: 8 total, 8 up, 8 in 2024-03-07T03:19:13.034 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:12 smithi145 bash[82329]: cluster 2024-03-07T03:19:12.303813+0000 mon.a (mon.0) 596 : cluster 0 osdmap e493: 8 total, 8 up, 8 in 2024-03-07T03:19:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:12 smithi189 bash[66969]: cluster 2024-03-07T03:19:11.780364+0000 mon.a (mon.0) 595 : cluster 0 osdmap e492: 8 total, 8 up, 8 in 2024-03-07T03:19:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:12 smithi189 bash[66969]: cluster 2024-03-07T03:19:12.303813+0000 mon.a (mon.0) 596 : cluster 0 osdmap e493: 8 total, 8 up, 8 in 2024-03-07T03:19:13.432 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.SnapshotPromoteDemote (9632 ms) 2024-03-07T03:19:13.432 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestMirroring.AioSnapshotCreate 2024-03-07T03:19:14.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:13 smithi145 bash[82329]: cluster 2024-03-07T03:19:12.618400+0000 mgr.y (mgr.34107) 191 : cluster 0 pgmap v339: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 264 B/s wr, 8 op/s 2024-03-07T03:19:14.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:13 smithi145 bash[82329]: cluster 2024-03-07T03:19:13.312849+0000 mon.a (mon.0) 597 : cluster 0 osdmap e494: 8 total, 8 up, 8 in 2024-03-07T03:19:14.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:13 smithi145 bash[82329]: audit 2024-03-07T03:19:13.423817+0000 mon.a (mon.0) 598 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/18a38a2b-5696-4311-b224-826a6547dbc3"}]: dispatch 2024-03-07T03:19:14.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:13 smithi145 bash[82329]: audit 2024-03-07T03:19:13.432308+0000 mon.a (mon.0) 599 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/18a38a2b-5696-4311-b224-826a6547dbc3"}]': finished 2024-03-07T03:19:14.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:13 smithi145 bash[79878]: cluster 2024-03-07T03:19:12.618400+0000 mgr.y (mgr.34107) 191 : cluster 0 pgmap v339: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 264 B/s wr, 8 op/s 2024-03-07T03:19:14.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:13 smithi145 bash[79878]: cluster 2024-03-07T03:19:13.312849+0000 mon.a (mon.0) 597 : cluster 0 osdmap e494: 8 total, 8 up, 8 in 2024-03-07T03:19:14.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:13 smithi145 bash[79878]: audit 2024-03-07T03:19:13.423817+0000 mon.a (mon.0) 598 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/18a38a2b-5696-4311-b224-826a6547dbc3"}]: dispatch 2024-03-07T03:19:14.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:13 smithi145 bash[79878]: audit 2024-03-07T03:19:13.432308+0000 mon.a (mon.0) 599 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/18a38a2b-5696-4311-b224-826a6547dbc3"}]': finished 2024-03-07T03:19:14.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:13 smithi189 bash[66969]: cluster 2024-03-07T03:19:12.618400+0000 mgr.y (mgr.34107) 191 : cluster 0 pgmap v339: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 264 B/s wr, 8 op/s 2024-03-07T03:19:14.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:13 smithi189 bash[66969]: cluster 2024-03-07T03:19:13.312849+0000 mon.a (mon.0) 597 : cluster 0 osdmap e494: 8 total, 8 up, 8 in 2024-03-07T03:19:14.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:13 smithi189 bash[66969]: audit 2024-03-07T03:19:13.423817+0000 mon.a (mon.0) 598 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/18a38a2b-5696-4311-b224-826a6547dbc3"}]: dispatch 2024-03-07T03:19:14.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:13 smithi189 bash[66969]: audit 2024-03-07T03:19:13.432308+0000 mon.a (mon.0) 599 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/18a38a2b-5696-4311-b224-826a6547dbc3"}]': finished 2024-03-07T03:19:14.304 INFO:teuthology.orchestra.run.smithi145.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:19:15.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: audit 2024-03-07T03:19:13.852484+0000 mon.a (mon.0) 600 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:15.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: audit 2024-03-07T03:19:13.864903+0000 mon.a (mon.0) 601 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:15.208 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: audit 2024-03-07T03:19:13.867747+0000 mon.a (mon.0) 602 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: audit 2024-03-07T03:19:13.869246+0000 mon.a (mon.0) 603 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: audit 2024-03-07T03:19:13.881147+0000 mon.a (mon.0) 604 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: audit 2024-03-07T03:19:13.960339+0000 mon.a (mon.0) 605 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: audit 2024-03-07T03:19:13.962628+0000 mon.a (mon.0) 606 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: audit 2024-03-07T03:19:13.964344+0000 mon.a (mon.0) 607 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: audit 2024-03-07T03:19:13.965448+0000 mon.a (mon.0) 608 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: audit 2024-03-07T03:19:13.966957+0000 mon.a (mon.0) 609 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: audit 2024-03-07T03:19:13.967121+0000 mgr.y (mgr.34107) 192 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: cephadm 2024-03-07T03:19:13.968083+0000 mgr.y (mgr.34107) 193 : cephadm 1 Upgrade: osd.1 is safe to restart 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: audit 2024-03-07T03:19:14.309339+0000 mgr.y (mgr.34107) 194 : audit 0 from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: cephadm 2024-03-07T03:19:14.309972+0000 mgr.y (mgr.34107) 195 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: audit 2024-03-07T03:19:14.315673+0000 mon.a (mon.0) 610 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: cluster 2024-03-07T03:19:14.430037+0000 mon.a (mon.0) 611 : cluster 0 osdmap e495: 8 total, 8 up, 8 in 2024-03-07T03:19:15.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[82329]: cluster 2024-03-07T03:19:14.619017+0000 mgr.y (mgr.34107) 196 : cluster 0 pgmap v342: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 9.6 KiB/s wr, 117 op/s 2024-03-07T03:19:15.210 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: audit 2024-03-07T03:19:13.852484+0000 mon.a (mon.0) 600 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:15.210 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: audit 2024-03-07T03:19:13.864903+0000 mon.a (mon.0) 601 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:15.210 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: audit 2024-03-07T03:19:13.867747+0000 mon.a (mon.0) 602 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:15.210 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: audit 2024-03-07T03:19:13.869246+0000 mon.a (mon.0) 603 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:15.210 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: audit 2024-03-07T03:19:13.881147+0000 mon.a (mon.0) 604 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:15.210 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: audit 2024-03-07T03:19:13.960339+0000 mon.a (mon.0) 605 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:15.210 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: audit 2024-03-07T03:19:13.962628+0000 mon.a (mon.0) 606 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:19:15.210 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: audit 2024-03-07T03:19:13.964344+0000 mon.a (mon.0) 607 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:19:15.210 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: audit 2024-03-07T03:19:13.965448+0000 mon.a (mon.0) 608 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:19:15.210 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: audit 2024-03-07T03:19:13.966957+0000 mon.a (mon.0) 609 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-07T03:19:15.210 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: audit 2024-03-07T03:19:13.967121+0000 mgr.y (mgr.34107) 192 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-07T03:19:15.210 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: cephadm 2024-03-07T03:19:13.968083+0000 mgr.y (mgr.34107) 193 : cephadm 1 Upgrade: osd.1 is safe to restart 2024-03-07T03:19:15.210 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: audit 2024-03-07T03:19:14.309339+0000 mgr.y (mgr.34107) 194 : audit 0 from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:19:15.211 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: cephadm 2024-03-07T03:19:14.309972+0000 mgr.y (mgr.34107) 195 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:19:15.211 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: audit 2024-03-07T03:19:14.315673+0000 mon.a (mon.0) 610 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:15.211 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: cluster 2024-03-07T03:19:14.430037+0000 mon.a (mon.0) 611 : cluster 0 osdmap e495: 8 total, 8 up, 8 in 2024-03-07T03:19:15.211 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:14 smithi145 bash[79878]: cluster 2024-03-07T03:19:14.619017+0000 mgr.y (mgr.34107) 196 : cluster 0 pgmap v342: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 9.6 KiB/s wr, 117 op/s 2024-03-07T03:19:15.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: audit 2024-03-07T03:19:13.852484+0000 mon.a (mon.0) 600 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:15.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: audit 2024-03-07T03:19:13.864903+0000 mon.a (mon.0) 601 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:15.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: audit 2024-03-07T03:19:13.867747+0000 mon.a (mon.0) 602 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:15.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: audit 2024-03-07T03:19:13.869246+0000 mon.a (mon.0) 603 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: audit 2024-03-07T03:19:13.881147+0000 mon.a (mon.0) 604 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: audit 2024-03-07T03:19:13.960339+0000 mon.a (mon.0) 605 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: audit 2024-03-07T03:19:13.962628+0000 mon.a (mon.0) 606 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:19:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: audit 2024-03-07T03:19:13.964344+0000 mon.a (mon.0) 607 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:19:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: audit 2024-03-07T03:19:13.965448+0000 mon.a (mon.0) 608 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:19:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: audit 2024-03-07T03:19:13.966957+0000 mon.a (mon.0) 609 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-07T03:19:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: audit 2024-03-07T03:19:13.967121+0000 mgr.y (mgr.34107) 192 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-07T03:19:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: cephadm 2024-03-07T03:19:13.968083+0000 mgr.y (mgr.34107) 193 : cephadm 1 Upgrade: osd.1 is safe to restart 2024-03-07T03:19:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: audit 2024-03-07T03:19:14.309339+0000 mgr.y (mgr.34107) 194 : audit 0 from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:19:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: cephadm 2024-03-07T03:19:14.309972+0000 mgr.y (mgr.34107) 195 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:19:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: audit 2024-03-07T03:19:14.315673+0000 mon.a (mon.0) 610 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: cluster 2024-03-07T03:19:14.430037+0000 mon.a (mon.0) 611 : cluster 0 osdmap e495: 8 total, 8 up, 8 in 2024-03-07T03:19:15.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:14 smithi189 bash[66969]: cluster 2024-03-07T03:19:14.619017+0000 mgr.y (mgr.34107) 196 : cluster 0 pgmap v342: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 9.6 KiB/s wr, 117 op/s 2024-03-07T03:19:15.872 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'sleep 60' 2024-03-07T03:19:16.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:16 smithi145 bash[79878]: cluster 2024-03-07T03:19:15.443435+0000 mon.a (mon.0) 612 : cluster 0 osdmap e496: 8 total, 8 up, 8 in 2024-03-07T03:19:16.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:16 smithi145 bash[82329]: cluster 2024-03-07T03:19:15.443435+0000 mon.a (mon.0) 612 : cluster 0 osdmap e496: 8 total, 8 up, 8 in 2024-03-07T03:19:16.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:16 smithi189 bash[66969]: cluster 2024-03-07T03:19:15.443435+0000 mon.a (mon.0) 612 : cluster 0 osdmap e496: 8 total, 8 up, 8 in 2024-03-07T03:19:17.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:17 smithi145 bash[79878]: cluster 2024-03-07T03:19:16.459739+0000 mon.a (mon.0) 613 : cluster 0 osdmap e497: 8 total, 8 up, 8 in 2024-03-07T03:19:17.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:17 smithi145 bash[79878]: cluster 2024-03-07T03:19:16.619656+0000 mgr.y (mgr.34107) 197 : cluster 0 pgmap v345: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 9.0 KiB/s wr, 105 op/s 2024-03-07T03:19:17.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:17 smithi145 bash[79878]: audit 2024-03-07T03:19:17.245646+0000 mon.a (mon.0) 614 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:17.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:17 smithi145 bash[79878]: audit 2024-03-07T03:19:17.249520+0000 mon.a (mon.0) 615 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:19:17.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:17 smithi145 bash[79878]: audit 2024-03-07T03:19:17.250614+0000 mon.a (mon.0) 616 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:17.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:17 smithi145 bash[79878]: cluster 2024-03-07T03:19:17.307462+0000 mon.a (mon.0) 617 : cluster 0 osdmap e498: 8 total, 8 up, 8 in 2024-03-07T03:19:17.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:17 smithi145 bash[82329]: cluster 2024-03-07T03:19:16.459739+0000 mon.a (mon.0) 613 : cluster 0 osdmap e497: 8 total, 8 up, 8 in 2024-03-07T03:19:17.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:17 smithi145 bash[82329]: cluster 2024-03-07T03:19:16.619656+0000 mgr.y (mgr.34107) 197 : cluster 0 pgmap v345: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 9.0 KiB/s wr, 105 op/s 2024-03-07T03:19:17.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:17 smithi145 bash[82329]: audit 2024-03-07T03:19:17.245646+0000 mon.a (mon.0) 614 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:17.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:17 smithi145 bash[82329]: audit 2024-03-07T03:19:17.249520+0000 mon.a (mon.0) 615 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:19:17.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:17 smithi145 bash[82329]: audit 2024-03-07T03:19:17.250614+0000 mon.a (mon.0) 616 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:17.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:17 smithi145 bash[82329]: cluster 2024-03-07T03:19:17.307462+0000 mon.a (mon.0) 617 : cluster 0 osdmap e498: 8 total, 8 up, 8 in 2024-03-07T03:19:17.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:17 smithi189 bash[66969]: cluster 2024-03-07T03:19:16.459739+0000 mon.a (mon.0) 613 : cluster 0 osdmap e497: 8 total, 8 up, 8 in 2024-03-07T03:19:17.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:17 smithi189 bash[66969]: cluster 2024-03-07T03:19:16.619656+0000 mgr.y (mgr.34107) 197 : cluster 0 pgmap v345: 105 pgs: 105 active+clean; 583 KiB data, 431 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s rd, 9.0 KiB/s wr, 105 op/s 2024-03-07T03:19:17.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:17 smithi189 bash[66969]: audit 2024-03-07T03:19:17.245646+0000 mon.a (mon.0) 614 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:17.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:17 smithi189 bash[66969]: audit 2024-03-07T03:19:17.249520+0000 mon.a (mon.0) 615 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:19:17.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:17 smithi189 bash[66969]: audit 2024-03-07T03:19:17.250614+0000 mon.a (mon.0) 616 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:17.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:17 smithi189 bash[66969]: cluster 2024-03-07T03:19:17.307462+0000 mon.a (mon.0) 617 : cluster 0 osdmap e498: 8 total, 8 up, 8 in 2024-03-07T03:19:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:18 smithi145 bash[79878]: cephadm 2024-03-07T03:19:17.238593+0000 mgr.y (mgr.34107) 198 : cephadm 1 Upgrade: Updating osd.1 2024-03-07T03:19:19.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:18 smithi145 bash[79878]: cephadm 2024-03-07T03:19:17.254746+0000 mgr.y (mgr.34107) 199 : cephadm 1 Deploying daemon osd.1 on smithi145 2024-03-07T03:19:19.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:18 smithi145 bash[79878]: audit 2024-03-07T03:19:17.643147+0000 mon.a (mon.0) 618 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:19.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:18 smithi145 bash[79878]: audit 2024-03-07T03:19:17.645209+0000 mon.a (mon.0) 619 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:19.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:18 smithi145 bash[79878]: cluster 2024-03-07T03:19:18.312554+0000 mon.a (mon.0) 620 : cluster 0 osdmap e499: 8 total, 8 up, 8 in 2024-03-07T03:19:19.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:18 smithi145 bash[79878]: audit 2024-03-07T03:19:18.591590+0000 mon.a (mon.0) 621 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:19.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:18 smithi145 bash[82329]: cephadm 2024-03-07T03:19:17.238593+0000 mgr.y (mgr.34107) 198 : cephadm 1 Upgrade: Updating osd.1 2024-03-07T03:19:19.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:18 smithi145 bash[82329]: cephadm 2024-03-07T03:19:17.254746+0000 mgr.y (mgr.34107) 199 : cephadm 1 Deploying daemon osd.1 on smithi145 2024-03-07T03:19:19.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:18 smithi145 bash[82329]: audit 2024-03-07T03:19:17.643147+0000 mon.a (mon.0) 618 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:19.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:18 smithi145 bash[82329]: audit 2024-03-07T03:19:17.645209+0000 mon.a (mon.0) 619 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:19.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:18 smithi145 bash[82329]: cluster 2024-03-07T03:19:18.312554+0000 mon.a (mon.0) 620 : cluster 0 osdmap e499: 8 total, 8 up, 8 in 2024-03-07T03:19:19.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:18 smithi145 bash[82329]: audit 2024-03-07T03:19:18.591590+0000 mon.a (mon.0) 621 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:18 smithi189 bash[66969]: cephadm 2024-03-07T03:19:17.238593+0000 mgr.y (mgr.34107) 198 : cephadm 1 Upgrade: Updating osd.1 2024-03-07T03:19:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:18 smithi189 bash[66969]: cephadm 2024-03-07T03:19:17.254746+0000 mgr.y (mgr.34107) 199 : cephadm 1 Deploying daemon osd.1 on smithi145 2024-03-07T03:19:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:18 smithi189 bash[66969]: audit 2024-03-07T03:19:17.643147+0000 mon.a (mon.0) 618 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:18 smithi189 bash[66969]: audit 2024-03-07T03:19:17.645209+0000 mon.a (mon.0) 619 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:18 smithi189 bash[66969]: cluster 2024-03-07T03:19:18.312554+0000 mon.a (mon.0) 620 : cluster 0 osdmap e499: 8 total, 8 up, 8 in 2024-03-07T03:19:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:18 smithi189 bash[66969]: audit 2024-03-07T03:19:18.591590+0000 mon.a (mon.0) 621 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:20.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:19 smithi145 bash[79878]: cluster 2024-03-07T03:19:18.620921+0000 mgr.y (mgr.34107) 200 : cluster 0 pgmap v348: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 14 KiB/s wr, 126 op/s 2024-03-07T03:19:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:19 smithi145 bash[79878]: cluster 2024-03-07T03:19:19.324518+0000 mon.a (mon.0) 622 : cluster 0 osdmap e500: 8 total, 8 up, 8 in 2024-03-07T03:19:20.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:19 smithi145 bash[82329]: cluster 2024-03-07T03:19:18.620921+0000 mgr.y (mgr.34107) 200 : cluster 0 pgmap v348: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 14 KiB/s wr, 126 op/s 2024-03-07T03:19:20.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:19 smithi145 bash[82329]: cluster 2024-03-07T03:19:19.324518+0000 mon.a (mon.0) 622 : cluster 0 osdmap e500: 8 total, 8 up, 8 in 2024-03-07T03:19:20.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:19 smithi189 bash[66969]: cluster 2024-03-07T03:19:18.620921+0000 mgr.y (mgr.34107) 200 : cluster 0 pgmap v348: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 14 KiB/s wr, 126 op/s 2024-03-07T03:19:20.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:19 smithi189 bash[66969]: cluster 2024-03-07T03:19:19.324518+0000 mon.a (mon.0) 622 : cluster 0 osdmap e500: 8 total, 8 up, 8 in 2024-03-07T03:19:20.767 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:19:21.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:21 smithi189 bash[66969]: cluster 2024-03-07T03:19:20.330464+0000 mon.a (mon.0) 623 : cluster 0 osdmap e501: 8 total, 8 up, 8 in 2024-03-07T03:19:21.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:21 smithi189 bash[66969]: cluster 2024-03-07T03:19:20.621649+0000 mgr.y (mgr.34107) 201 : cluster 0 pgmap v351: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 14 KiB/s wr, 126 op/s 2024-03-07T03:19:21.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:21 smithi145 bash[79878]: cluster 2024-03-07T03:19:20.330464+0000 mon.a (mon.0) 623 : cluster 0 osdmap e501: 8 total, 8 up, 8 in 2024-03-07T03:19:21.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:21 smithi145 bash[79878]: cluster 2024-03-07T03:19:20.621649+0000 mgr.y (mgr.34107) 201 : cluster 0 pgmap v351: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 14 KiB/s wr, 126 op/s 2024-03-07T03:19:21.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:21 smithi145 bash[82329]: cluster 2024-03-07T03:19:20.330464+0000 mon.a (mon.0) 623 : cluster 0 osdmap e501: 8 total, 8 up, 8 in 2024-03-07T03:19:21.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:21 smithi145 bash[82329]: cluster 2024-03-07T03:19:20.621649+0000 mgr.y (mgr.34107) 201 : cluster 0 pgmap v351: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 14 KiB/s wr, 126 op/s 2024-03-07T03:19:22.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:19:22 smithi189 bash[59814]: ts=2024-03-07T03:19:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:22 smithi189 bash[66969]: cluster 2024-03-07T03:19:21.336973+0000 mon.a (mon.0) 624 : cluster 0 osdmap e502: 8 total, 8 up, 8 in 2024-03-07T03:19:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:22 smithi189 bash[66969]: cluster 2024-03-07T03:19:22.311614+0000 mon.a (mon.0) 625 : cluster 0 osdmap e503: 8 total, 8 up, 8 in 2024-03-07T03:19:22.586 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:22 smithi145 bash[82329]: cluster 2024-03-07T03:19:21.336973+0000 mon.a (mon.0) 624 : cluster 0 osdmap e502: 8 total, 8 up, 8 in 2024-03-07T03:19:22.586 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:22 smithi145 bash[82329]: cluster 2024-03-07T03:19:22.311614+0000 mon.a (mon.0) 625 : cluster 0 osdmap e503: 8 total, 8 up, 8 in 2024-03-07T03:19:22.587 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:22 smithi145 bash[79878]: cluster 2024-03-07T03:19:21.336973+0000 mon.a (mon.0) 624 : cluster 0 osdmap e502: 8 total, 8 up, 8 in 2024-03-07T03:19:22.587 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:22 smithi145 bash[79878]: cluster 2024-03-07T03:19:22.311614+0000 mon.a (mon.0) 625 : cluster 0 osdmap e503: 8 total, 8 up, 8 in 2024-03-07T03:19:22.905 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:19:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:19:22] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.43.0" 2024-03-07T03:19:23.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:22 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:23.310 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:19:22 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:23.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:22 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:23.311 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:19:22 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:23.311 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:19:22 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:23.311 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:19:22 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:23.311 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:19:22 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:23.312 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:19:22 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:23.312 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:22 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:23.312 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:23 smithi145 systemd[1]: Stopping Ceph osd.1 for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:19:23.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:23 smithi145 bash[79878]: cluster 2024-03-07T03:19:22.622125+0000 mgr.y (mgr.34107) 202 : cluster 0 pgmap v354: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-03-07T03:19:23.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:23 smithi145 bash[82329]: cluster 2024-03-07T03:19:22.622125+0000 mgr.y (mgr.34107) 202 : cluster 0 pgmap v354: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-03-07T03:19:23.810 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:23 smithi145 bash[39308]: debug 2024-03-07T03:19:23.526+0000 7f6aaebad700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:19:23.810 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:23 smithi145 bash[39308]: debug 2024-03-07T03:19:23.526+0000 7f6aaebad700 -1 osd.1 504 *** Got signal Terminated *** 2024-03-07T03:19:23.810 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:23 smithi145 bash[39308]: debug 2024-03-07T03:19:23.526+0000 7f6aaebad700 -1 osd.1 504 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:19:23.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:23 smithi189 bash[66969]: cluster 2024-03-07T03:19:22.622125+0000 mgr.y (mgr.34107) 202 : cluster 0 pgmap v354: 105 pgs: 105 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-03-07T03:19:24.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:24 smithi145 bash[82329]: cluster 2024-03-07T03:19:23.350979+0000 mon.a (mon.0) 626 : cluster 0 osdmap e504: 8 total, 8 up, 8 in 2024-03-07T03:19:24.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:24 smithi145 bash[82329]: cluster 2024-03-07T03:19:23.531897+0000 mon.a (mon.0) 627 : cluster 1 osd.1 marked itself down and dead 2024-03-07T03:19:24.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:24 smithi145 bash[79878]: debug 2024-03-07T03:19:24.342+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:19:24.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:24 smithi145 bash[79878]: cluster 2024-03-07T03:19:23.350979+0000 mon.a (mon.0) 626 : cluster 0 osdmap e504: 8 total, 8 up, 8 in 2024-03-07T03:19:24.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:24 smithi145 bash[79878]: cluster 2024-03-07T03:19:23.531897+0000 mon.a (mon.0) 627 : cluster 1 osd.1 marked itself down and dead 2024-03-07T03:19:24.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:24 smithi189 bash[66969]: cluster 2024-03-07T03:19:23.350979+0000 mon.a (mon.0) 626 : cluster 0 osdmap e504: 8 total, 8 up, 8 in 2024-03-07T03:19:24.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:24 smithi189 bash[66969]: cluster 2024-03-07T03:19:23.531897+0000 mon.a (mon.0) 627 : cluster 1 osd.1 marked itself down and dead 2024-03-07T03:19:25.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:25 smithi145 bash[79878]: cluster 2024-03-07T03:19:24.346382+0000 mon.a (mon.0) 628 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:19:25.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:25 smithi145 bash[79878]: cluster 2024-03-07T03:19:24.346442+0000 mon.a (mon.0) 629 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:19:25.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:25 smithi145 bash[79878]: cluster 2024-03-07T03:19:24.362675+0000 mon.a (mon.0) 630 : cluster 0 osdmap e505: 8 total, 7 up, 8 in 2024-03-07T03:19:25.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:25 smithi145 bash[79878]: cluster 2024-03-07T03:19:24.622680+0000 mgr.y (mgr.34107) 203 : cluster 0 pgmap v357: 105 pgs: 12 stale+active+clean, 93 active+clean; 584 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 11 KiB/s wr, 96 op/s 2024-03-07T03:19:25.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:25 smithi145 bash[82329]: cluster 2024-03-07T03:19:24.346382+0000 mon.a (mon.0) 628 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:19:25.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:25 smithi145 bash[82329]: cluster 2024-03-07T03:19:24.346442+0000 mon.a (mon.0) 629 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:19:25.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:25 smithi145 bash[82329]: cluster 2024-03-07T03:19:24.362675+0000 mon.a (mon.0) 630 : cluster 0 osdmap e505: 8 total, 7 up, 8 in 2024-03-07T03:19:25.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:25 smithi145 bash[82329]: cluster 2024-03-07T03:19:24.622680+0000 mgr.y (mgr.34107) 203 : cluster 0 pgmap v357: 105 pgs: 12 stale+active+clean, 93 active+clean; 584 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 11 KiB/s wr, 96 op/s 2024-03-07T03:19:25.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:25 smithi189 bash[66969]: cluster 2024-03-07T03:19:24.346382+0000 mon.a (mon.0) 628 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:19:25.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:25 smithi189 bash[66969]: cluster 2024-03-07T03:19:24.346442+0000 mon.a (mon.0) 629 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:19:25.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:25 smithi189 bash[66969]: cluster 2024-03-07T03:19:24.362675+0000 mon.a (mon.0) 630 : cluster 0 osdmap e505: 8 total, 7 up, 8 in 2024-03-07T03:19:25.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:25 smithi189 bash[66969]: cluster 2024-03-07T03:19:24.622680+0000 mgr.y (mgr.34107) 203 : cluster 0 pgmap v357: 105 pgs: 12 stale+active+clean, 93 active+clean; 584 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 11 KiB/s wr, 96 op/s 2024-03-07T03:19:25.855 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:25 smithi145 bash[93415]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-1 2024-03-07T03:19:25.855 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:25 smithi145 bash[93533]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-1 2024-03-07T03:19:26.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:26 smithi145 bash[79878]: cluster 2024-03-07T03:19:25.381461+0000 mon.a (mon.0) 631 : cluster 0 osdmap e506: 8 total, 7 up, 8 in 2024-03-07T03:19:26.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:26 smithi145 bash[82329]: cluster 2024-03-07T03:19:25.381461+0000 mon.a (mon.0) 631 : cluster 0 osdmap e506: 8 total, 7 up, 8 in 2024-03-07T03:19:26.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:26 smithi189 bash[66969]: cluster 2024-03-07T03:19:25.381461+0000 mon.a (mon.0) 631 : cluster 0 osdmap e506: 8 total, 7 up, 8 in 2024-03-07T03:19:27.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:27 smithi145 bash[79878]: cluster 2024-03-07T03:19:26.385498+0000 mon.a (mon.0) 632 : cluster 0 osdmap e507: 8 total, 7 up, 8 in 2024-03-07T03:19:27.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:27 smithi145 bash[79878]: cluster 2024-03-07T03:19:26.623337+0000 mgr.y (mgr.34107) 204 : cluster 0 pgmap v360: 105 pgs: 12 stale+active+clean, 93 active+clean; 584 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 11 KiB/s wr, 94 op/s 2024-03-07T03:19:27.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:27 smithi145 bash[82329]: cluster 2024-03-07T03:19:26.385498+0000 mon.a (mon.0) 632 : cluster 0 osdmap e507: 8 total, 7 up, 8 in 2024-03-07T03:19:27.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:27 smithi145 bash[82329]: cluster 2024-03-07T03:19:26.623337+0000 mgr.y (mgr.34107) 204 : cluster 0 pgmap v360: 105 pgs: 12 stale+active+clean, 93 active+clean; 584 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 11 KiB/s wr, 94 op/s 2024-03-07T03:19:27.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:27 smithi189 bash[66969]: cluster 2024-03-07T03:19:26.385498+0000 mon.a (mon.0) 632 : cluster 0 osdmap e507: 8 total, 7 up, 8 in 2024-03-07T03:19:27.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:27 smithi189 bash[66969]: cluster 2024-03-07T03:19:26.623337+0000 mgr.y (mgr.34107) 204 : cluster 0 pgmap v360: 105 pgs: 12 stale+active+clean, 93 active+clean; 584 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 11 KiB/s wr, 94 op/s 2024-03-07T03:19:28.248 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:28 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:28.248 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:28 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:28.248 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:19:28 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:28.248 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:19:28 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:28.249 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:27 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.1.service: Deactivated successfully. 2024-03-07T03:19:28.249 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:28 smithi145 systemd[1]: Stopped Ceph osd.1 for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:19:28.249 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:28 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:28.249 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:19:28 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:28.249 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:19:28 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:28.250 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:19:28 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:28.250 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:19:28 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:28.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:28 smithi145 bash[82329]: cluster 2024-03-07T03:19:27.394003+0000 mon.a (mon.0) 633 : cluster 0 osdmap e508: 8 total, 7 up, 8 in 2024-03-07T03:19:28.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:28 smithi145 bash[82329]: audit 2024-03-07T03:19:28.306183+0000 mon.a (mon.0) 634 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:28.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:28 smithi145 bash[82329]: audit 2024-03-07T03:19:28.313818+0000 mon.a (mon.0) 635 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:28.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:28 smithi145 bash[82329]: audit 2024-03-07T03:19:28.352973+0000 mon.a (mon.0) 636 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:28.560 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:28 smithi145 systemd[1]: Started Ceph osd.1 for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:19:28.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:28 smithi145 bash[79878]: cluster 2024-03-07T03:19:27.394003+0000 mon.a (mon.0) 633 : cluster 0 osdmap e508: 8 total, 7 up, 8 in 2024-03-07T03:19:28.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:28 smithi145 bash[79878]: audit 2024-03-07T03:19:28.306183+0000 mon.a (mon.0) 634 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:28.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:28 smithi145 bash[79878]: audit 2024-03-07T03:19:28.313818+0000 mon.a (mon.0) 635 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:28.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:28 smithi145 bash[79878]: audit 2024-03-07T03:19:28.352973+0000 mon.a (mon.0) 636 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:28 smithi189 bash[66969]: cluster 2024-03-07T03:19:27.394003+0000 mon.a (mon.0) 633 : cluster 0 osdmap e508: 8 total, 7 up, 8 in 2024-03-07T03:19:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:28 smithi189 bash[66969]: audit 2024-03-07T03:19:28.306183+0000 mon.a (mon.0) 634 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:28 smithi189 bash[66969]: audit 2024-03-07T03:19:28.313818+0000 mon.a (mon.0) 635 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:28 smithi189 bash[66969]: audit 2024-03-07T03:19:28.352973+0000 mon.a (mon.0) 636 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:29.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:29 smithi145 bash[79878]: cluster 2024-03-07T03:19:28.401089+0000 mon.a (mon.0) 637 : cluster 0 osdmap e509: 8 total, 7 up, 8 in 2024-03-07T03:19:29.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:29 smithi145 bash[79878]: cluster 2024-03-07T03:19:28.624125+0000 mgr.y (mgr.34107) 205 : cluster 0 pgmap v363: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 160 KiB/s rd, 19 KiB/s wr, 217 op/s; 78/480 objects degraded (16.250%) 2024-03-07T03:19:29.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:29 smithi145 bash[82329]: cluster 2024-03-07T03:19:28.401089+0000 mon.a (mon.0) 637 : cluster 0 osdmap e509: 8 total, 7 up, 8 in 2024-03-07T03:19:29.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:29 smithi145 bash[82329]: cluster 2024-03-07T03:19:28.624125+0000 mgr.y (mgr.34107) 205 : cluster 0 pgmap v363: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 160 KiB/s rd, 19 KiB/s wr, 217 op/s; 78/480 objects degraded (16.250%) 2024-03-07T03:19:29.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:29 smithi189 bash[66969]: cluster 2024-03-07T03:19:28.401089+0000 mon.a (mon.0) 637 : cluster 0 osdmap e509: 8 total, 7 up, 8 in 2024-03-07T03:19:29.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:29 smithi189 bash[66969]: cluster 2024-03-07T03:19:28.624125+0000 mgr.y (mgr.34107) 205 : cluster 0 pgmap v363: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 160 KiB/s rd, 19 KiB/s wr, 217 op/s; 78/480 objects degraded (16.250%) 2024-03-07T03:19:30.179 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:29 smithi145 bash[93684]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-07T03:19:30.179 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:29 smithi145 bash[93684]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-07T03:19:30.179 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:29 smithi145 bash[93684]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-07T03:19:30.179 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:29 smithi145 bash[93684]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-07T03:19:30.179 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:29 smithi145 bash[93684]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-07T03:19:30.179 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:29 smithi145 bash[93684]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-07T03:19:30.180 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:29 smithi145 bash[93684]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-07T03:19:30.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:30 smithi145 bash[79878]: cluster 2024-03-07T03:19:29.395297+0000 mon.a (mon.0) 638 : cluster 3 Health check failed: Degraded data redundancy: 78/480 objects degraded (16.250%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:30.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:30 smithi145 bash[79878]: cluster 2024-03-07T03:19:29.412896+0000 mon.a (mon.0) 639 : cluster 0 osdmap e510: 8 total, 7 up, 8 in 2024-03-07T03:19:30.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:30 smithi145 bash[82329]: cluster 2024-03-07T03:19:29.395297+0000 mon.a (mon.0) 638 : cluster 3 Health check failed: Degraded data redundancy: 78/480 objects degraded (16.250%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:30.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:30 smithi145 bash[82329]: cluster 2024-03-07T03:19:29.412896+0000 mon.a (mon.0) 639 : cluster 0 osdmap e510: 8 total, 7 up, 8 in 2024-03-07T03:19:30.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:30 smithi189 bash[66969]: cluster 2024-03-07T03:19:29.395297+0000 mon.a (mon.0) 638 : cluster 3 Health check failed: Degraded data redundancy: 78/480 objects degraded (16.250%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:30.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:30 smithi189 bash[66969]: cluster 2024-03-07T03:19:29.412896+0000 mon.a (mon.0) 639 : cluster 0 osdmap e510: 8 total, 7 up, 8 in 2024-03-07T03:19:31.754 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:31 smithi145 bash[79878]: cluster 2024-03-07T03:19:30.418747+0000 mon.a (mon.0) 640 : cluster 0 osdmap e511: 8 total, 7 up, 8 in 2024-03-07T03:19:31.754 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:31 smithi145 bash[79878]: cluster 2024-03-07T03:19:30.625035+0000 mgr.y (mgr.34107) 206 : cluster 0 pgmap v366: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 160 KiB/s rd, 19 KiB/s wr, 217 op/s; 78/480 objects degraded (16.250%) 2024-03-07T03:19:31.754 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:31 smithi145 bash[82329]: cluster 2024-03-07T03:19:30.418747+0000 mon.a (mon.0) 640 : cluster 0 osdmap e511: 8 total, 7 up, 8 in 2024-03-07T03:19:31.754 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:31 smithi145 bash[82329]: cluster 2024-03-07T03:19:30.625035+0000 mgr.y (mgr.34107) 206 : cluster 0 pgmap v366: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 160 KiB/s rd, 19 KiB/s wr, 217 op/s; 78/480 objects degraded (16.250%) 2024-03-07T03:19:31.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:31 smithi189 bash[66969]: cluster 2024-03-07T03:19:30.418747+0000 mon.a (mon.0) 640 : cluster 0 osdmap e511: 8 total, 7 up, 8 in 2024-03-07T03:19:31.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:31 smithi189 bash[66969]: cluster 2024-03-07T03:19:30.625035+0000 mgr.y (mgr.34107) 206 : cluster 0 pgmap v366: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 160 KiB/s rd, 19 KiB/s wr, 217 op/s; 78/480 objects degraded (16.250%) 2024-03-07T03:19:32.059 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:31 smithi145 bash[93904]: debug 2024-03-07T03:19:31.838+0000 7f3b2b267700 1 -- 172.21.15.145:0/1750653718 <== mon.0 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56038b0a3680 con 0x56038b0df000 2024-03-07T03:19:32.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:32 smithi189 bash[66969]: cluster 2024-03-07T03:19:31.429022+0000 mon.a (mon.0) 641 : cluster 0 osdmap e512: 8 total, 7 up, 8 in 2024-03-07T03:19:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:19:32 smithi189 bash[59814]: ts=2024-03-07T03:19:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:32.750 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:32 smithi145 bash[93904]: debug 2024-03-07T03:19:32.378+0000 7f3b343ff7c0 -1 Falling back to public interface 2024-03-07T03:19:32.750 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:32 smithi145 bash[82329]: cluster 2024-03-07T03:19:31.429022+0000 mon.a (mon.0) 641 : cluster 0 osdmap e512: 8 total, 7 up, 8 in 2024-03-07T03:19:32.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:32 smithi145 bash[79878]: cluster 2024-03-07T03:19:31.429022+0000 mon.a (mon.0) 641 : cluster 0 osdmap e512: 8 total, 7 up, 8 in 2024-03-07T03:19:33.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:19:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:19:32] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.43.0" 2024-03-07T03:19:33.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:33 smithi145 bash[79878]: cluster 2024-03-07T03:19:32.436689+0000 mon.a (mon.0) 642 : cluster 0 osdmap e513: 8 total, 7 up, 8 in 2024-03-07T03:19:33.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:33 smithi145 bash[79878]: cluster 2024-03-07T03:19:32.625912+0000 mgr.y (mgr.34107) 207 : cluster 0 pgmap v369: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 0 B/s wr, 2 op/s; 76/474 objects degraded (16.034%) 2024-03-07T03:19:33.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:33 smithi145 bash[79878]: audit 2024-03-07T03:19:32.645893+0000 mon.a (mon.0) 643 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:33.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:33 smithi145 bash[79878]: audit 2024-03-07T03:19:32.648233+0000 mon.a (mon.0) 644 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:33.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:33 smithi145 bash[82329]: cluster 2024-03-07T03:19:32.436689+0000 mon.a (mon.0) 642 : cluster 0 osdmap e513: 8 total, 7 up, 8 in 2024-03-07T03:19:33.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:33 smithi145 bash[82329]: cluster 2024-03-07T03:19:32.625912+0000 mgr.y (mgr.34107) 207 : cluster 0 pgmap v369: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 0 B/s wr, 2 op/s; 76/474 objects degraded (16.034%) 2024-03-07T03:19:33.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:33 smithi145 bash[82329]: audit 2024-03-07T03:19:32.645893+0000 mon.a (mon.0) 643 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:33.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:33 smithi145 bash[82329]: audit 2024-03-07T03:19:32.648233+0000 mon.a (mon.0) 644 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:33.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:33 smithi189 bash[66969]: cluster 2024-03-07T03:19:32.436689+0000 mon.a (mon.0) 642 : cluster 0 osdmap e513: 8 total, 7 up, 8 in 2024-03-07T03:19:33.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:33 smithi189 bash[66969]: cluster 2024-03-07T03:19:32.625912+0000 mgr.y (mgr.34107) 207 : cluster 0 pgmap v369: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 0 B/s wr, 2 op/s; 76/474 objects degraded (16.034%) 2024-03-07T03:19:33.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:33 smithi189 bash[66969]: audit 2024-03-07T03:19:32.645893+0000 mon.a (mon.0) 643 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:33.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:33 smithi189 bash[66969]: audit 2024-03-07T03:19:32.648233+0000 mon.a (mon.0) 644 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:34.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:34 smithi145 bash[79878]: cluster 2024-03-07T03:19:33.447413+0000 mon.a (mon.0) 645 : cluster 0 osdmap e514: 8 total, 7 up, 8 in 2024-03-07T03:19:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:34 smithi145 bash[82329]: cluster 2024-03-07T03:19:33.447413+0000 mon.a (mon.0) 645 : cluster 0 osdmap e514: 8 total, 7 up, 8 in 2024-03-07T03:19:34.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:34 smithi189 bash[66969]: cluster 2024-03-07T03:19:33.447413+0000 mon.a (mon.0) 645 : cluster 0 osdmap e514: 8 total, 7 up, 8 in 2024-03-07T03:19:35.809 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:35 smithi145 bash[93904]: debug 2024-03-07T03:19:35.533+0000 7f3b343ff7c0 -1 osd.1 504 log_to_monitors true 2024-03-07T03:19:35.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:35 smithi145 bash[79878]: cluster 2024-03-07T03:19:34.449083+0000 mon.a (mon.0) 646 : cluster 0 osdmap e515: 8 total, 7 up, 8 in 2024-03-07T03:19:35.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:35 smithi145 bash[79878]: cluster 2024-03-07T03:19:34.626422+0000 mgr.y (mgr.34107) 208 : cluster 0 pgmap v372: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 7.5 KiB/s wr, 132 op/s; 68/426 objects degraded (15.962%) 2024-03-07T03:19:35.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:35 smithi145 bash[79878]: audit 2024-03-07T03:19:35.096910+0000 mon.a (mon.0) 647 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:35.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:35 smithi145 bash[79878]: audit 2024-03-07T03:19:35.102561+0000 mon.a (mon.0) 648 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:35.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:35 smithi145 bash[82329]: cluster 2024-03-07T03:19:34.449083+0000 mon.a (mon.0) 646 : cluster 0 osdmap e515: 8 total, 7 up, 8 in 2024-03-07T03:19:35.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:35 smithi145 bash[82329]: cluster 2024-03-07T03:19:34.626422+0000 mgr.y (mgr.34107) 208 : cluster 0 pgmap v372: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 7.5 KiB/s wr, 132 op/s; 68/426 objects degraded (15.962%) 2024-03-07T03:19:35.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:35 smithi145 bash[82329]: audit 2024-03-07T03:19:35.096910+0000 mon.a (mon.0) 647 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:35.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:35 smithi145 bash[82329]: audit 2024-03-07T03:19:35.102561+0000 mon.a (mon.0) 648 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:35.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:35 smithi189 bash[66969]: cluster 2024-03-07T03:19:34.449083+0000 mon.a (mon.0) 646 : cluster 0 osdmap e515: 8 total, 7 up, 8 in 2024-03-07T03:19:35.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:35 smithi189 bash[66969]: cluster 2024-03-07T03:19:34.626422+0000 mgr.y (mgr.34107) 208 : cluster 0 pgmap v372: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 7.5 KiB/s wr, 132 op/s; 68/426 objects degraded (15.962%) 2024-03-07T03:19:35.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:35 smithi189 bash[66969]: audit 2024-03-07T03:19:35.096910+0000 mon.a (mon.0) 647 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:35.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:35 smithi189 bash[66969]: audit 2024-03-07T03:19:35.102561+0000 mon.a (mon.0) 648 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:36.809 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:19:36 smithi145 bash[93904]: debug 2024-03-07T03:19:36.497+0000 7f3b25857700 -1 osd.1 504 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-07T03:19:36.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:36 smithi145 bash[79878]: cluster 2024-03-07T03:19:35.446096+0000 mon.a (mon.0) 649 : cluster 3 Health check update: Degraded data redundancy: 68/426 objects degraded (15.962%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:36.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:36 smithi145 bash[79878]: cluster 2024-03-07T03:19:35.458001+0000 mon.a (mon.0) 650 : cluster 0 osdmap e516: 8 total, 7 up, 8 in 2024-03-07T03:19:36.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:36 smithi145 bash[79878]: audit 2024-03-07T03:19:35.543404+0000 mon.c (mon.2) 4 : audit 1 from='osd.1 [v2:172.21.15.145:6810/183115113,v1:172.21.15.145:6811/183115113]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:19:36.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:36 smithi145 bash[79878]: audit 2024-03-07T03:19:35.543738+0000 mon.a (mon.0) 651 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:19:36.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:36 smithi145 bash[79878]: audit 2024-03-07T03:19:36.045860+0000 mon.a (mon.0) 652 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:36.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:36 smithi145 bash[79878]: audit 2024-03-07T03:19:36.056529+0000 mon.a (mon.0) 653 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:36.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:36 smithi145 bash[82329]: cluster 2024-03-07T03:19:35.446096+0000 mon.a (mon.0) 649 : cluster 3 Health check update: Degraded data redundancy: 68/426 objects degraded (15.962%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:36.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:36 smithi145 bash[82329]: cluster 2024-03-07T03:19:35.458001+0000 mon.a (mon.0) 650 : cluster 0 osdmap e516: 8 total, 7 up, 8 in 2024-03-07T03:19:36.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:36 smithi145 bash[82329]: audit 2024-03-07T03:19:35.543404+0000 mon.c (mon.2) 4 : audit 1 from='osd.1 [v2:172.21.15.145:6810/183115113,v1:172.21.15.145:6811/183115113]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:19:36.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:36 smithi145 bash[82329]: audit 2024-03-07T03:19:35.543738+0000 mon.a (mon.0) 651 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:19:36.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:36 smithi145 bash[82329]: audit 2024-03-07T03:19:36.045860+0000 mon.a (mon.0) 652 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:36.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:36 smithi145 bash[82329]: audit 2024-03-07T03:19:36.056529+0000 mon.a (mon.0) 653 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:36.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:36 smithi189 bash[66969]: cluster 2024-03-07T03:19:35.446096+0000 mon.a (mon.0) 649 : cluster 3 Health check update: Degraded data redundancy: 68/426 objects degraded (15.962%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:36.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:36 smithi189 bash[66969]: cluster 2024-03-07T03:19:35.458001+0000 mon.a (mon.0) 650 : cluster 0 osdmap e516: 8 total, 7 up, 8 in 2024-03-07T03:19:36.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:36 smithi189 bash[66969]: audit 2024-03-07T03:19:35.543404+0000 mon.c (mon.2) 4 : audit 1 from='osd.1 [v2:172.21.15.145:6810/183115113,v1:172.21.15.145:6811/183115113]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:19:36.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:36 smithi189 bash[66969]: audit 2024-03-07T03:19:35.543738+0000 mon.a (mon.0) 651 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:19:36.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:36 smithi189 bash[66969]: audit 2024-03-07T03:19:36.045860+0000 mon.a (mon.0) 652 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:36.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:36 smithi189 bash[66969]: audit 2024-03-07T03:19:36.056529+0000 mon.a (mon.0) 653 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:37.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:37 smithi145 bash[82329]: audit 2024-03-07T03:19:36.461774+0000 mon.a (mon.0) 654 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-07T03:19:37.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:37 smithi145 bash[82329]: cluster 2024-03-07T03:19:36.465403+0000 mon.a (mon.0) 655 : cluster 0 osdmap e517: 8 total, 7 up, 8 in 2024-03-07T03:19:37.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:37 smithi145 bash[82329]: audit 2024-03-07T03:19:36.466099+0000 mon.c (mon.2) 5 : audit 1 from='osd.1 [v2:172.21.15.145:6810/183115113,v1:172.21.15.145:6811/183115113]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:19:37.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:37 smithi145 bash[82329]: audit 2024-03-07T03:19:36.467042+0000 mon.a (mon.0) 656 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:19:37.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:37 smithi145 bash[82329]: cluster 2024-03-07T03:19:36.627246+0000 mgr.y (mgr.34107) 209 : cluster 0 pgmap v375: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 7.5 KiB/s wr, 129 op/s; 68/426 objects degraded (15.962%) 2024-03-07T03:19:37.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:37 smithi145 bash[79878]: debug 2024-03-07T03:19:37.461+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:19:37.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:37 smithi145 bash[79878]: audit 2024-03-07T03:19:36.461774+0000 mon.a (mon.0) 654 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-07T03:19:37.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:37 smithi145 bash[79878]: cluster 2024-03-07T03:19:36.465403+0000 mon.a (mon.0) 655 : cluster 0 osdmap e517: 8 total, 7 up, 8 in 2024-03-07T03:19:37.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:37 smithi145 bash[79878]: audit 2024-03-07T03:19:36.466099+0000 mon.c (mon.2) 5 : audit 1 from='osd.1 [v2:172.21.15.145:6810/183115113,v1:172.21.15.145:6811/183115113]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:19:37.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:37 smithi145 bash[79878]: audit 2024-03-07T03:19:36.467042+0000 mon.a (mon.0) 656 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:19:37.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:37 smithi145 bash[79878]: cluster 2024-03-07T03:19:36.627246+0000 mgr.y (mgr.34107) 209 : cluster 0 pgmap v375: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 7.5 KiB/s wr, 129 op/s; 68/426 objects degraded (15.962%) 2024-03-07T03:19:37.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:37 smithi189 bash[66969]: audit 2024-03-07T03:19:36.461774+0000 mon.a (mon.0) 654 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-07T03:19:37.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:37 smithi189 bash[66969]: cluster 2024-03-07T03:19:36.465403+0000 mon.a (mon.0) 655 : cluster 0 osdmap e517: 8 total, 7 up, 8 in 2024-03-07T03:19:37.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:37 smithi189 bash[66969]: audit 2024-03-07T03:19:36.466099+0000 mon.c (mon.2) 5 : audit 1 from='osd.1 [v2:172.21.15.145:6810/183115113,v1:172.21.15.145:6811/183115113]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:19:37.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:37 smithi189 bash[66969]: audit 2024-03-07T03:19:36.467042+0000 mon.a (mon.0) 656 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:19:37.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:37 smithi189 bash[66969]: cluster 2024-03-07T03:19:36.627246+0000 mgr.y (mgr.34107) 209 : cluster 0 pgmap v375: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 584 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 7.5 KiB/s wr, 129 op/s; 68/426 objects degraded (15.962%) 2024-03-07T03:19:38.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:38 smithi145 bash[79878]: cluster 2024-03-07T03:19:37.463749+0000 mon.a (mon.0) 657 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:19:38.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:38 smithi145 bash[79878]: cluster 2024-03-07T03:19:37.463808+0000 mon.a (mon.0) 658 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:19:38.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:38 smithi145 bash[79878]: cluster 2024-03-07T03:19:37.483698+0000 mon.a (mon.0) 659 : cluster 1 osd.1 [v2:172.21.15.145:6810/183115113,v1:172.21.15.145:6811/183115113] boot 2024-03-07T03:19:38.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:38 smithi145 bash[79878]: cluster 2024-03-07T03:19:37.483870+0000 mon.a (mon.0) 660 : cluster 0 osdmap e518: 8 total, 8 up, 8 in 2024-03-07T03:19:38.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:38 smithi145 bash[79878]: audit 2024-03-07T03:19:37.486956+0000 mon.a (mon.0) 661 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:19:38.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:38 smithi145 bash[82329]: cluster 2024-03-07T03:19:37.463749+0000 mon.a (mon.0) 657 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:19:38.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:38 smithi145 bash[82329]: cluster 2024-03-07T03:19:37.463808+0000 mon.a (mon.0) 658 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:19:38.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:38 smithi145 bash[82329]: cluster 2024-03-07T03:19:37.483698+0000 mon.a (mon.0) 659 : cluster 1 osd.1 [v2:172.21.15.145:6810/183115113,v1:172.21.15.145:6811/183115113] boot 2024-03-07T03:19:38.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:38 smithi145 bash[82329]: cluster 2024-03-07T03:19:37.483870+0000 mon.a (mon.0) 660 : cluster 0 osdmap e518: 8 total, 8 up, 8 in 2024-03-07T03:19:38.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:38 smithi145 bash[82329]: audit 2024-03-07T03:19:37.486956+0000 mon.a (mon.0) 661 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:19:38.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:38 smithi189 bash[66969]: cluster 2024-03-07T03:19:37.463749+0000 mon.a (mon.0) 657 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:19:38.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:38 smithi189 bash[66969]: cluster 2024-03-07T03:19:37.463808+0000 mon.a (mon.0) 658 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:19:38.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:38 smithi189 bash[66969]: cluster 2024-03-07T03:19:37.483698+0000 mon.a (mon.0) 659 : cluster 1 osd.1 [v2:172.21.15.145:6810/183115113,v1:172.21.15.145:6811/183115113] boot 2024-03-07T03:19:38.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:38 smithi189 bash[66969]: cluster 2024-03-07T03:19:37.483870+0000 mon.a (mon.0) 660 : cluster 0 osdmap e518: 8 total, 8 up, 8 in 2024-03-07T03:19:38.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:38 smithi189 bash[66969]: audit 2024-03-07T03:19:37.486956+0000 mon.a (mon.0) 661 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:19:39.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:39 smithi145 bash[79878]: cluster 2024-03-07T03:19:38.485850+0000 mon.a (mon.0) 662 : cluster 0 osdmap e519: 8 total, 8 up, 8 in 2024-03-07T03:19:39.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:39 smithi145 bash[79878]: cluster 2024-03-07T03:19:38.628032+0000 mgr.y (mgr.34107) 210 : cluster 0 pgmap v378: 105 pgs: 79 active+clean, 10 active+undersized, 1 active+recovery_wait, 7 active+undersized+degraded, 5 active+recovery_wait+degraded, 1 active+recovering+degraded, 2 active+clean+snaptrim_wait; 583 KiB data, 430 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 6.5 KiB/s wr, 97 op/s; 45/393 objects degraded (11.450%); 25 B/s, 22 keys/s, 1 objects/s recovering 2024-03-07T03:19:39.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:39 smithi145 bash[82329]: cluster 2024-03-07T03:19:38.485850+0000 mon.a (mon.0) 662 : cluster 0 osdmap e519: 8 total, 8 up, 8 in 2024-03-07T03:19:39.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:39 smithi145 bash[82329]: cluster 2024-03-07T03:19:38.628032+0000 mgr.y (mgr.34107) 210 : cluster 0 pgmap v378: 105 pgs: 79 active+clean, 10 active+undersized, 1 active+recovery_wait, 7 active+undersized+degraded, 5 active+recovery_wait+degraded, 1 active+recovering+degraded, 2 active+clean+snaptrim_wait; 583 KiB data, 430 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 6.5 KiB/s wr, 97 op/s; 45/393 objects degraded (11.450%); 25 B/s, 22 keys/s, 1 objects/s recovering 2024-03-07T03:19:39.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:39 smithi189 bash[66969]: cluster 2024-03-07T03:19:38.485850+0000 mon.a (mon.0) 662 : cluster 0 osdmap e519: 8 total, 8 up, 8 in 2024-03-07T03:19:39.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:39 smithi189 bash[66969]: cluster 2024-03-07T03:19:38.628032+0000 mgr.y (mgr.34107) 210 : cluster 0 pgmap v378: 105 pgs: 79 active+clean, 10 active+undersized, 1 active+recovery_wait, 7 active+undersized+degraded, 5 active+recovery_wait+degraded, 1 active+recovering+degraded, 2 active+clean+snaptrim_wait; 583 KiB data, 430 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 6.5 KiB/s wr, 97 op/s; 45/393 objects degraded (11.450%); 25 B/s, 22 keys/s, 1 objects/s recovering 2024-03-07T03:19:40.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:40 smithi145 bash[79878]: cluster 2024-03-07T03:19:39.501327+0000 mon.a (mon.0) 663 : cluster 0 osdmap e520: 8 total, 8 up, 8 in 2024-03-07T03:19:40.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:40 smithi145 bash[82329]: cluster 2024-03-07T03:19:39.501327+0000 mon.a (mon.0) 663 : cluster 0 osdmap e520: 8 total, 8 up, 8 in 2024-03-07T03:19:40.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:40 smithi189 bash[66969]: cluster 2024-03-07T03:19:39.501327+0000 mon.a (mon.0) 663 : cluster 0 osdmap e520: 8 total, 8 up, 8 in 2024-03-07T03:19:41.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:41 smithi145 bash[79878]: cluster 2024-03-07T03:19:40.509897+0000 mon.a (mon.0) 664 : cluster 0 osdmap e521: 8 total, 8 up, 8 in 2024-03-07T03:19:41.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:41 smithi145 bash[79878]: cluster 2024-03-07T03:19:40.628689+0000 mgr.y (mgr.34107) 211 : cluster 0 pgmap v381: 105 pgs: 79 active+clean, 10 active+undersized, 1 active+recovery_wait, 7 active+undersized+degraded, 5 active+recovery_wait+degraded, 1 active+recovering+degraded, 2 active+clean+snaptrim_wait; 583 KiB data, 430 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 6.5 KiB/s wr, 97 op/s; 45/393 objects degraded (11.450%); 25 B/s, 22 keys/s, 1 objects/s recovering 2024-03-07T03:19:41.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:41 smithi145 bash[82329]: cluster 2024-03-07T03:19:40.509897+0000 mon.a (mon.0) 664 : cluster 0 osdmap e521: 8 total, 8 up, 8 in 2024-03-07T03:19:41.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:41 smithi145 bash[82329]: cluster 2024-03-07T03:19:40.628689+0000 mgr.y (mgr.34107) 211 : cluster 0 pgmap v381: 105 pgs: 79 active+clean, 10 active+undersized, 1 active+recovery_wait, 7 active+undersized+degraded, 5 active+recovery_wait+degraded, 1 active+recovering+degraded, 2 active+clean+snaptrim_wait; 583 KiB data, 430 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 6.5 KiB/s wr, 97 op/s; 45/393 objects degraded (11.450%); 25 B/s, 22 keys/s, 1 objects/s recovering 2024-03-07T03:19:41.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:41 smithi189 bash[66969]: cluster 2024-03-07T03:19:40.509897+0000 mon.a (mon.0) 664 : cluster 0 osdmap e521: 8 total, 8 up, 8 in 2024-03-07T03:19:41.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:41 smithi189 bash[66969]: cluster 2024-03-07T03:19:40.628689+0000 mgr.y (mgr.34107) 211 : cluster 0 pgmap v381: 105 pgs: 79 active+clean, 10 active+undersized, 1 active+recovery_wait, 7 active+undersized+degraded, 5 active+recovery_wait+degraded, 1 active+recovering+degraded, 2 active+clean+snaptrim_wait; 583 KiB data, 430 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 6.5 KiB/s wr, 97 op/s; 45/393 objects degraded (11.450%); 25 B/s, 22 keys/s, 1 objects/s recovering 2024-03-07T03:19:42.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:19:42 smithi189 bash[59814]: ts=2024-03-07T03:19:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:42.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:42 smithi189 bash[66969]: cluster 2024-03-07T03:19:41.521629+0000 mon.a (mon.0) 665 : cluster 0 osdmap e522: 8 total, 8 up, 8 in 2024-03-07T03:19:42.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:42 smithi189 bash[66969]: cluster 2024-03-07T03:19:42.311701+0000 mon.a (mon.0) 666 : cluster 3 Health check update: Degraded data redundancy: 45/393 objects degraded (11.450%), 13 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:42.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:42 smithi145 bash[79878]: cluster 2024-03-07T03:19:41.521629+0000 mon.a (mon.0) 665 : cluster 0 osdmap e522: 8 total, 8 up, 8 in 2024-03-07T03:19:42.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:42 smithi145 bash[79878]: cluster 2024-03-07T03:19:42.311701+0000 mon.a (mon.0) 666 : cluster 3 Health check update: Degraded data redundancy: 45/393 objects degraded (11.450%), 13 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:42.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:19:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:19:42] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.43.0" 2024-03-07T03:19:42.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:42 smithi145 bash[82329]: cluster 2024-03-07T03:19:41.521629+0000 mon.a (mon.0) 665 : cluster 0 osdmap e522: 8 total, 8 up, 8 in 2024-03-07T03:19:42.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:42 smithi145 bash[82329]: cluster 2024-03-07T03:19:42.311701+0000 mon.a (mon.0) 666 : cluster 3 Health check update: Degraded data redundancy: 45/393 objects degraded (11.450%), 13 pgs degraded (PG_DEGRADED) 2024-03-07T03:19:43.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:43 smithi145 bash[79878]: cluster 2024-03-07T03:19:42.524776+0000 mon.a (mon.0) 667 : cluster 0 osdmap e523: 8 total, 8 up, 8 in 2024-03-07T03:19:43.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:43 smithi145 bash[79878]: cluster 2024-03-07T03:19:42.629339+0000 mgr.y (mgr.34107) 212 : cluster 0 pgmap v384: 105 pgs: 96 active+clean, 1 active+recovery_wait, 5 active+recovery_wait+degraded, 1 active+recovering+degraded, 2 active+clean+snaptrim_wait; 583 KiB data, 430 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 2.0 KiB/s wr, 37 op/s; 13/372 objects degraded (3.495%); 26 B/s, 7 keys/s, 2 objects/s recovering 2024-03-07T03:19:43.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:43 smithi145 bash[82329]: cluster 2024-03-07T03:19:42.524776+0000 mon.a (mon.0) 667 : cluster 0 osdmap e523: 8 total, 8 up, 8 in 2024-03-07T03:19:43.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:43 smithi145 bash[82329]: cluster 2024-03-07T03:19:42.629339+0000 mgr.y (mgr.34107) 212 : cluster 0 pgmap v384: 105 pgs: 96 active+clean, 1 active+recovery_wait, 5 active+recovery_wait+degraded, 1 active+recovering+degraded, 2 active+clean+snaptrim_wait; 583 KiB data, 430 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 2.0 KiB/s wr, 37 op/s; 13/372 objects degraded (3.495%); 26 B/s, 7 keys/s, 2 objects/s recovering 2024-03-07T03:19:43.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:43 smithi189 bash[66969]: cluster 2024-03-07T03:19:42.524776+0000 mon.a (mon.0) 667 : cluster 0 osdmap e523: 8 total, 8 up, 8 in 2024-03-07T03:19:43.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:43 smithi189 bash[66969]: cluster 2024-03-07T03:19:42.629339+0000 mgr.y (mgr.34107) 212 : cluster 0 pgmap v384: 105 pgs: 96 active+clean, 1 active+recovery_wait, 5 active+recovery_wait+degraded, 1 active+recovering+degraded, 2 active+clean+snaptrim_wait; 583 KiB data, 430 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 2.0 KiB/s wr, 37 op/s; 13/372 objects degraded (3.495%); 26 B/s, 7 keys/s, 2 objects/s recovering 2024-03-07T03:19:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:44 smithi145 bash[79878]: cluster 2024-03-07T03:19:43.541684+0000 mon.a (mon.0) 668 : cluster 0 osdmap e524: 8 total, 8 up, 8 in 2024-03-07T03:19:44.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:44 smithi145 bash[82329]: cluster 2024-03-07T03:19:43.541684+0000 mon.a (mon.0) 668 : cluster 0 osdmap e524: 8 total, 8 up, 8 in 2024-03-07T03:19:44.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:44 smithi189 bash[66969]: cluster 2024-03-07T03:19:43.541684+0000 mon.a (mon.0) 668 : cluster 0 osdmap e524: 8 total, 8 up, 8 in 2024-03-07T03:19:45.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:45 smithi145 bash[79878]: cluster 2024-03-07T03:19:44.543705+0000 mon.a (mon.0) 669 : cluster 0 osdmap e525: 8 total, 8 up, 8 in 2024-03-07T03:19:45.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:45 smithi145 bash[79878]: cluster 2024-03-07T03:19:44.629834+0000 mgr.y (mgr.34107) 213 : cluster 0 pgmap v387: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 7.0 KiB/s wr, 115 op/s; 39 B/s, 7 keys/s, 5 objects/s recovering 2024-03-07T03:19:45.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:45 smithi145 bash[82329]: cluster 2024-03-07T03:19:44.543705+0000 mon.a (mon.0) 669 : cluster 0 osdmap e525: 8 total, 8 up, 8 in 2024-03-07T03:19:45.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:45 smithi145 bash[82329]: cluster 2024-03-07T03:19:44.629834+0000 mgr.y (mgr.34107) 213 : cluster 0 pgmap v387: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 7.0 KiB/s wr, 115 op/s; 39 B/s, 7 keys/s, 5 objects/s recovering 2024-03-07T03:19:45.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:45 smithi189 bash[66969]: cluster 2024-03-07T03:19:44.543705+0000 mon.a (mon.0) 669 : cluster 0 osdmap e525: 8 total, 8 up, 8 in 2024-03-07T03:19:45.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:45 smithi189 bash[66969]: cluster 2024-03-07T03:19:44.629834+0000 mgr.y (mgr.34107) 213 : cluster 0 pgmap v387: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 7.0 KiB/s wr, 115 op/s; 39 B/s, 7 keys/s, 5 objects/s recovering 2024-03-07T03:19:46.685 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestMirroring.AioSnapshotCreate (33252 ms) 2024-03-07T03:19:46.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:46 smithi145 bash[79878]: cluster 2024-03-07T03:19:45.549618+0000 mon.a (mon.0) 670 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/372 objects degraded (3.495%), 6 pgs degraded) 2024-03-07T03:19:46.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:46 smithi145 bash[79878]: cluster 2024-03-07T03:19:45.563142+0000 mon.a (mon.0) 671 : cluster 0 osdmap e526: 8 total, 8 up, 8 in 2024-03-07T03:19:46.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:46 smithi145 bash[82329]: cluster 2024-03-07T03:19:45.549618+0000 mon.a (mon.0) 670 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/372 objects degraded (3.495%), 6 pgs degraded) 2024-03-07T03:19:46.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:46 smithi145 bash[82329]: cluster 2024-03-07T03:19:45.563142+0000 mon.a (mon.0) 671 : cluster 0 osdmap e526: 8 total, 8 up, 8 in 2024-03-07T03:19:46.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:46 smithi189 bash[66969]: cluster 2024-03-07T03:19:45.549618+0000 mon.a (mon.0) 670 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/372 objects degraded (3.495%), 6 pgs degraded) 2024-03-07T03:19:46.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:46 smithi189 bash[66969]: cluster 2024-03-07T03:19:45.563142+0000 mon.a (mon.0) 671 : cluster 0 osdmap e526: 8 total, 8 up, 8 in 2024-03-07T03:19:47.329 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 51 tests from TestMirroring (97459 ms total) 2024-03-07T03:19:47.329 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:19:47.329 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 34 tests from TestInternal 2024-03-07T03:19:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:47 smithi189 bash[66969]: cluster 2024-03-07T03:19:46.573763+0000 mon.a (mon.0) 672 : cluster 0 osdmap e527: 8 total, 8 up, 8 in 2024-03-07T03:19:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:47 smithi189 bash[66969]: cluster 2024-03-07T03:19:46.630457+0000 mgr.y (mgr.34107) 214 : cluster 0 pgmap v390: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 7.0 KiB/s wr, 125 op/s; 12 B/s, 3 objects/s recovering 2024-03-07T03:19:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:47 smithi189 bash[66969]: audit 2024-03-07T03:19:46.681036+0000 mon.a (mon.0) 673 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/2632b036-6fb9-4d08-a0cf-64eba29750fe"}]: dispatch 2024-03-07T03:19:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:47 smithi189 bash[66969]: audit 2024-03-07T03:19:46.684904+0000 mon.a (mon.0) 674 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/2632b036-6fb9-4d08-a0cf-64eba29750fe"}]': finished 2024-03-07T03:19:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:47 smithi189 bash[66969]: audit 2024-03-07T03:19:46.705608+0000 mon.a (mon.0) 675 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:47 smithi189 bash[66969]: audit 2024-03-07T03:19:46.713506+0000 mon.a (mon.0) 676 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:47 smithi189 bash[66969]: audit 2024-03-07T03:19:46.715411+0000 mon.a (mon.0) 677 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:47 smithi189 bash[66969]: audit 2024-03-07T03:19:46.716644+0000 mon.a (mon.0) 678 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:47 smithi189 bash[66969]: audit 2024-03-07T03:19:46.724401+0000 mon.a (mon.0) 679 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:48.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:47 smithi189 bash[66969]: audit 2024-03-07T03:19:46.812833+0000 mon.a (mon.0) 680 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:48.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:47 smithi189 bash[66969]: cluster 2024-03-07T03:19:47.324371+0000 mon.a (mon.0) 681 : cluster 0 osdmap e528: 8 total, 8 up, 8 in 2024-03-07T03:19:48.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[79878]: cluster 2024-03-07T03:19:46.573763+0000 mon.a (mon.0) 672 : cluster 0 osdmap e527: 8 total, 8 up, 8 in 2024-03-07T03:19:48.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[79878]: cluster 2024-03-07T03:19:46.630457+0000 mgr.y (mgr.34107) 214 : cluster 0 pgmap v390: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 7.0 KiB/s wr, 125 op/s; 12 B/s, 3 objects/s recovering 2024-03-07T03:19:48.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[79878]: audit 2024-03-07T03:19:46.681036+0000 mon.a (mon.0) 673 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/2632b036-6fb9-4d08-a0cf-64eba29750fe"}]: dispatch 2024-03-07T03:19:48.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[79878]: audit 2024-03-07T03:19:46.684904+0000 mon.a (mon.0) 674 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/2632b036-6fb9-4d08-a0cf-64eba29750fe"}]': finished 2024-03-07T03:19:48.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[79878]: audit 2024-03-07T03:19:46.705608+0000 mon.a (mon.0) 675 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:48.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[79878]: audit 2024-03-07T03:19:46.713506+0000 mon.a (mon.0) 676 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:48.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[79878]: audit 2024-03-07T03:19:46.715411+0000 mon.a (mon.0) 677 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:48.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[79878]: audit 2024-03-07T03:19:46.716644+0000 mon.a (mon.0) 678 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:48.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[79878]: audit 2024-03-07T03:19:46.724401+0000 mon.a (mon.0) 679 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:48.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[79878]: audit 2024-03-07T03:19:46.812833+0000 mon.a (mon.0) 680 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:48.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[79878]: cluster 2024-03-07T03:19:47.324371+0000 mon.a (mon.0) 681 : cluster 0 osdmap e528: 8 total, 8 up, 8 in 2024-03-07T03:19:48.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[82329]: cluster 2024-03-07T03:19:46.573763+0000 mon.a (mon.0) 672 : cluster 0 osdmap e527: 8 total, 8 up, 8 in 2024-03-07T03:19:48.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[82329]: cluster 2024-03-07T03:19:46.630457+0000 mgr.y (mgr.34107) 214 : cluster 0 pgmap v390: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 7.0 KiB/s wr, 125 op/s; 12 B/s, 3 objects/s recovering 2024-03-07T03:19:48.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[82329]: audit 2024-03-07T03:19:46.681036+0000 mon.a (mon.0) 673 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/2632b036-6fb9-4d08-a0cf-64eba29750fe"}]: dispatch 2024-03-07T03:19:48.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[82329]: audit 2024-03-07T03:19:46.684904+0000 mon.a (mon.0) 674 : audit 1 from='client.? 172.21.15.145:0/2868967712' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/2632b036-6fb9-4d08-a0cf-64eba29750fe"}]': finished 2024-03-07T03:19:48.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[82329]: audit 2024-03-07T03:19:46.705608+0000 mon.a (mon.0) 675 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:48.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[82329]: audit 2024-03-07T03:19:46.713506+0000 mon.a (mon.0) 676 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:48.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[82329]: audit 2024-03-07T03:19:46.715411+0000 mon.a (mon.0) 677 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:48.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[82329]: audit 2024-03-07T03:19:46.716644+0000 mon.a (mon.0) 678 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:48.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[82329]: audit 2024-03-07T03:19:46.724401+0000 mon.a (mon.0) 679 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:48.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[82329]: audit 2024-03-07T03:19:46.812833+0000 mon.a (mon.0) 680 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:48.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:47 smithi145 bash[82329]: cluster 2024-03-07T03:19:47.324371+0000 mon.a (mon.0) 681 : cluster 0 osdmap e528: 8 total, 8 up, 8 in 2024-03-07T03:19:49.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:48 smithi189 bash[66969]: audit 2024-03-07T03:19:47.833220+0000 mon.a (mon.0) 682 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:48 smithi189 bash[66969]: audit 2024-03-07T03:19:47.835284+0000 mon.a (mon.0) 683 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:48 smithi189 bash[66969]: cluster 2024-03-07T03:19:48.341954+0000 mon.a (mon.0) 684 : cluster 0 osdmap e529: 8 total, 8 up, 8 in 2024-03-07T03:19:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:48 smithi189 bash[66969]: audit 2024-03-07T03:19:48.357257+0000 mon.c (mon.2) 6 : audit 1 from='client.? 172.21.15.145:0/2654189882' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:19:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:48 smithi189 bash[66969]: audit 2024-03-07T03:19:48.358122+0000 mon.a (mon.0) 685 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:19:49.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:48 smithi145 bash[79878]: audit 2024-03-07T03:19:47.833220+0000 mon.a (mon.0) 682 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:49.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:48 smithi145 bash[79878]: audit 2024-03-07T03:19:47.835284+0000 mon.a (mon.0) 683 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:49.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:48 smithi145 bash[79878]: cluster 2024-03-07T03:19:48.341954+0000 mon.a (mon.0) 684 : cluster 0 osdmap e529: 8 total, 8 up, 8 in 2024-03-07T03:19:49.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:48 smithi145 bash[79878]: audit 2024-03-07T03:19:48.357257+0000 mon.c (mon.2) 6 : audit 1 from='client.? 172.21.15.145:0/2654189882' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:19:49.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:48 smithi145 bash[79878]: audit 2024-03-07T03:19:48.358122+0000 mon.a (mon.0) 685 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:19:49.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:48 smithi145 bash[82329]: audit 2024-03-07T03:19:47.833220+0000 mon.a (mon.0) 682 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:49.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:48 smithi145 bash[82329]: audit 2024-03-07T03:19:47.835284+0000 mon.a (mon.0) 683 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:49.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:48 smithi145 bash[82329]: cluster 2024-03-07T03:19:48.341954+0000 mon.a (mon.0) 684 : cluster 0 osdmap e529: 8 total, 8 up, 8 in 2024-03-07T03:19:49.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:48 smithi145 bash[82329]: audit 2024-03-07T03:19:48.357257+0000 mon.c (mon.2) 6 : audit 1 from='client.? 172.21.15.145:0/2654189882' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:19:49.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:48 smithi145 bash[82329]: audit 2024-03-07T03:19:48.358122+0000 mon.a (mon.0) 685 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:19:49.342 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.OpenByID 2024-03-07T03:19:50.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:49 smithi189 bash[66969]: cluster 2024-03-07T03:19:48.631573+0000 mgr.y (mgr.34107) 215 : cluster 0 pgmap v393: 105 pgs: 23 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 414 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:50.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:49 smithi189 bash[66969]: cluster 2024-03-07T03:19:49.335762+0000 mon.a (mon.0) 686 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:19:50.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:49 smithi189 bash[66969]: audit 2024-03-07T03:19:49.345033+0000 mon.a (mon.0) 687 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:19:50.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:49 smithi189 bash[66969]: cluster 2024-03-07T03:19:49.352927+0000 mon.a (mon.0) 688 : cluster 0 osdmap e530: 8 total, 8 up, 8 in 2024-03-07T03:19:50.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:49 smithi145 bash[79878]: cluster 2024-03-07T03:19:48.631573+0000 mgr.y (mgr.34107) 215 : cluster 0 pgmap v393: 105 pgs: 23 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 414 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:50.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:49 smithi145 bash[79878]: cluster 2024-03-07T03:19:49.335762+0000 mon.a (mon.0) 686 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:19:50.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:49 smithi145 bash[79878]: audit 2024-03-07T03:19:49.345033+0000 mon.a (mon.0) 687 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:19:50.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:49 smithi145 bash[79878]: cluster 2024-03-07T03:19:49.352927+0000 mon.a (mon.0) 688 : cluster 0 osdmap e530: 8 total, 8 up, 8 in 2024-03-07T03:19:50.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:49 smithi145 bash[82329]: cluster 2024-03-07T03:19:48.631573+0000 mgr.y (mgr.34107) 215 : cluster 0 pgmap v393: 105 pgs: 23 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 414 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:50.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:49 smithi145 bash[82329]: cluster 2024-03-07T03:19:49.335762+0000 mon.a (mon.0) 686 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:19:50.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:49 smithi145 bash[82329]: audit 2024-03-07T03:19:49.345033+0000 mon.a (mon.0) 687 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:19:50.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:49 smithi145 bash[82329]: cluster 2024-03-07T03:19:49.352927+0000 mon.a (mon.0) 688 : cluster 0 osdmap e530: 8 total, 8 up, 8 in 2024-03-07T03:19:51.417 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.OpenByID (2074 ms) 2024-03-07T03:19:51.417 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.OpenSnapDNE 2024-03-07T03:19:51.463 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.OpenSnapDNE (46 ms) 2024-03-07T03:19:51.463 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.IsExclusiveLockOwner 2024-03-07T03:19:51.513 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.IsExclusiveLockOwner (50 ms) 2024-03-07T03:19:51.513 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.ResizeLocksImage 2024-03-07T03:19:51.583 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.ResizeLocksImage (70 ms) 2024-03-07T03:19:51.583 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.ResizeFailsToLockImage 2024-03-07T03:19:51.641 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.ResizeFailsToLockImage (58 ms) 2024-03-07T03:19:51.641 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.SnapCreateLocksImage 2024-03-07T03:19:51.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:51 smithi145 bash[79878]: cluster 2024-03-07T03:19:50.352986+0000 mon.a (mon.0) 689 : cluster 0 osdmap e531: 8 total, 8 up, 8 in 2024-03-07T03:19:51.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:51 smithi145 bash[79878]: cluster 2024-03-07T03:19:50.632266+0000 mgr.y (mgr.34107) 216 : cluster 0 pgmap v396: 105 pgs: 23 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 414 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:51.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:51 smithi145 bash[82329]: cluster 2024-03-07T03:19:50.352986+0000 mon.a (mon.0) 689 : cluster 0 osdmap e531: 8 total, 8 up, 8 in 2024-03-07T03:19:51.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:51 smithi145 bash[82329]: cluster 2024-03-07T03:19:50.632266+0000 mgr.y (mgr.34107) 216 : cluster 0 pgmap v396: 105 pgs: 23 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 414 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:51.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:51 smithi189 bash[66969]: cluster 2024-03-07T03:19:50.352986+0000 mon.a (mon.0) 689 : cluster 0 osdmap e531: 8 total, 8 up, 8 in 2024-03-07T03:19:51.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:51 smithi189 bash[66969]: cluster 2024-03-07T03:19:50.632266+0000 mgr.y (mgr.34107) 216 : cluster 0 pgmap v396: 105 pgs: 23 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 414 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:52.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:19:52 smithi189 bash[59814]: ts=2024-03-07T03:19:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:52.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:52 smithi189 bash[66969]: cluster 2024-03-07T03:19:51.347789+0000 mon.a (mon.0) 690 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:19:52.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:52 smithi189 bash[66969]: cluster 2024-03-07T03:19:51.369527+0000 mon.a (mon.0) 691 : cluster 0 osdmap e532: 8 total, 8 up, 8 in 2024-03-07T03:19:52.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:52 smithi145 bash[79878]: cluster 2024-03-07T03:19:51.347789+0000 mon.a (mon.0) 690 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:19:52.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:52 smithi145 bash[79878]: cluster 2024-03-07T03:19:51.369527+0000 mon.a (mon.0) 691 : cluster 0 osdmap e532: 8 total, 8 up, 8 in 2024-03-07T03:19:52.752 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:52 smithi145 bash[82329]: cluster 2024-03-07T03:19:51.347789+0000 mon.a (mon.0) 690 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:19:52.752 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:52 smithi145 bash[82329]: cluster 2024-03-07T03:19:51.369527+0000 mon.a (mon.0) 691 : cluster 0 osdmap e532: 8 total, 8 up, 8 in 2024-03-07T03:19:53.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:19:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:19:52] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.43.0" 2024-03-07T03:19:53.398 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.SnapCreateLocksImage (1757 ms) 2024-03-07T03:19:53.398 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.SnapCreateFailsToLockImage 2024-03-07T03:19:53.442 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.SnapCreateFailsToLockImage (44 ms) 2024-03-07T03:19:53.442 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.SnapRollbackLocksImage 2024-03-07T03:19:53.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:53 smithi145 bash[79878]: cluster 2024-03-07T03:19:52.367316+0000 mon.a (mon.0) 692 : cluster 0 osdmap e533: 8 total, 8 up, 8 in 2024-03-07T03:19:53.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:53 smithi145 bash[79878]: cluster 2024-03-07T03:19:52.632982+0000 mgr.y (mgr.34107) 217 : cluster 0 pgmap v399: 105 pgs: 23 creating+peering, 82 active+clean; 582 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 2.7 KiB/s wr, 32 op/s 2024-03-07T03:19:53.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:53 smithi145 bash[79878]: audit 2024-03-07T03:19:52.912175+0000 mon.a (mon.0) 693 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:53.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:53 smithi145 bash[79878]: audit 2024-03-07T03:19:52.924217+0000 mon.a (mon.0) 694 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:53.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:53 smithi145 bash[82329]: cluster 2024-03-07T03:19:52.367316+0000 mon.a (mon.0) 692 : cluster 0 osdmap e533: 8 total, 8 up, 8 in 2024-03-07T03:19:53.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:53 smithi145 bash[82329]: cluster 2024-03-07T03:19:52.632982+0000 mgr.y (mgr.34107) 217 : cluster 0 pgmap v399: 105 pgs: 23 creating+peering, 82 active+clean; 582 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 2.7 KiB/s wr, 32 op/s 2024-03-07T03:19:53.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:53 smithi145 bash[82329]: audit 2024-03-07T03:19:52.912175+0000 mon.a (mon.0) 693 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:53.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:53 smithi145 bash[82329]: audit 2024-03-07T03:19:52.924217+0000 mon.a (mon.0) 694 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:53.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:53 smithi189 bash[66969]: cluster 2024-03-07T03:19:52.367316+0000 mon.a (mon.0) 692 : cluster 0 osdmap e533: 8 total, 8 up, 8 in 2024-03-07T03:19:53.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:53 smithi189 bash[66969]: cluster 2024-03-07T03:19:52.632982+0000 mgr.y (mgr.34107) 217 : cluster 0 pgmap v399: 105 pgs: 23 creating+peering, 82 active+clean; 582 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 2.7 KiB/s wr, 32 op/s 2024-03-07T03:19:53.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:53 smithi189 bash[66969]: audit 2024-03-07T03:19:52.912175+0000 mon.a (mon.0) 693 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:53.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:53 smithi189 bash[66969]: audit 2024-03-07T03:19:52.924217+0000 mon.a (mon.0) 694 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:54.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:54 smithi145 bash[79878]: cluster 2024-03-07T03:19:53.386270+0000 mon.a (mon.0) 695 : cluster 0 osdmap e534: 8 total, 8 up, 8 in 2024-03-07T03:19:54.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:54 smithi145 bash[79878]: audit 2024-03-07T03:19:53.603253+0000 mon.a (mon.0) 696 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:54.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:54 smithi145 bash[79878]: audit 2024-03-07T03:19:53.605164+0000 mon.a (mon.0) 697 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:54.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:54 smithi145 bash[79878]: audit 2024-03-07T03:19:53.616740+0000 mon.a (mon.0) 698 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:54.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:54 smithi145 bash[82329]: cluster 2024-03-07T03:19:53.386270+0000 mon.a (mon.0) 695 : cluster 0 osdmap e534: 8 total, 8 up, 8 in 2024-03-07T03:19:54.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:54 smithi145 bash[82329]: audit 2024-03-07T03:19:53.603253+0000 mon.a (mon.0) 696 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:54.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:54 smithi145 bash[82329]: audit 2024-03-07T03:19:53.605164+0000 mon.a (mon.0) 697 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:54.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:54 smithi145 bash[82329]: audit 2024-03-07T03:19:53.616740+0000 mon.a (mon.0) 698 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:54.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:54 smithi189 bash[66969]: cluster 2024-03-07T03:19:53.386270+0000 mon.a (mon.0) 695 : cluster 0 osdmap e534: 8 total, 8 up, 8 in 2024-03-07T03:19:54.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:54 smithi189 bash[66969]: audit 2024-03-07T03:19:53.603253+0000 mon.a (mon.0) 696 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:54.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:54 smithi189 bash[66969]: audit 2024-03-07T03:19:53.605164+0000 mon.a (mon.0) 697 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:54.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:54 smithi189 bash[66969]: audit 2024-03-07T03:19:53.616740+0000 mon.a (mon.0) 698 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:19:55.426 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.SnapRollbackLocksImage (1984 ms) 2024-03-07T03:19:55.426 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.SnapRollbackFailsToLockImage 2024-03-07T03:19:55.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:55 smithi145 bash[79878]: cluster 2024-03-07T03:19:54.395833+0000 mon.a (mon.0) 699 : cluster 0 osdmap e535: 8 total, 8 up, 8 in 2024-03-07T03:19:55.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:55 smithi145 bash[79878]: cluster 2024-03-07T03:19:54.633766+0000 mgr.y (mgr.34107) 218 : cluster 0 pgmap v402: 105 pgs: 105 active+clean; 582 KiB data, 415 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 9.2 KiB/s wr, 113 op/s 2024-03-07T03:19:55.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:55 smithi145 bash[82329]: cluster 2024-03-07T03:19:54.395833+0000 mon.a (mon.0) 699 : cluster 0 osdmap e535: 8 total, 8 up, 8 in 2024-03-07T03:19:55.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:55 smithi145 bash[82329]: cluster 2024-03-07T03:19:54.633766+0000 mgr.y (mgr.34107) 218 : cluster 0 pgmap v402: 105 pgs: 105 active+clean; 582 KiB data, 415 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 9.2 KiB/s wr, 113 op/s 2024-03-07T03:19:55.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:55 smithi189 bash[66969]: cluster 2024-03-07T03:19:54.395833+0000 mon.a (mon.0) 699 : cluster 0 osdmap e535: 8 total, 8 up, 8 in 2024-03-07T03:19:55.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:55 smithi189 bash[66969]: cluster 2024-03-07T03:19:54.633766+0000 mgr.y (mgr.34107) 218 : cluster 0 pgmap v402: 105 pgs: 105 active+clean; 582 KiB data, 415 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 9.2 KiB/s wr, 113 op/s 2024-03-07T03:19:56.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:56 smithi145 bash[79878]: cluster 2024-03-07T03:19:55.407051+0000 mon.a (mon.0) 700 : cluster 0 osdmap e536: 8 total, 8 up, 8 in 2024-03-07T03:19:56.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:56 smithi145 bash[82329]: cluster 2024-03-07T03:19:55.407051+0000 mon.a (mon.0) 700 : cluster 0 osdmap e536: 8 total, 8 up, 8 in 2024-03-07T03:19:56.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:56 smithi189 bash[66969]: cluster 2024-03-07T03:19:55.407051+0000 mon.a (mon.0) 700 : cluster 0 osdmap e536: 8 total, 8 up, 8 in 2024-03-07T03:19:57.358 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.SnapRollbackFailsToLockImage (1932 ms) 2024-03-07T03:19:57.358 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.SnapSetReleasesLock 2024-03-07T03:19:57.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:57 smithi145 bash[79878]: cluster 2024-03-07T03:19:56.410036+0000 mon.a (mon.0) 701 : cluster 0 osdmap e537: 8 total, 8 up, 8 in 2024-03-07T03:19:57.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:57 smithi145 bash[79878]: cluster 2024-03-07T03:19:56.634620+0000 mgr.y (mgr.34107) 219 : cluster 0 pgmap v405: 105 pgs: 105 active+clean; 582 KiB data, 415 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 8.2 KiB/s wr, 87 op/s 2024-03-07T03:19:57.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:57 smithi145 bash[79878]: cluster 2024-03-07T03:19:57.329459+0000 mon.a (mon.0) 702 : cluster 0 osdmap e538: 8 total, 8 up, 8 in 2024-03-07T03:19:57.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:57 smithi145 bash[82329]: cluster 2024-03-07T03:19:56.410036+0000 mon.a (mon.0) 701 : cluster 0 osdmap e537: 8 total, 8 up, 8 in 2024-03-07T03:19:57.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:57 smithi145 bash[82329]: cluster 2024-03-07T03:19:56.634620+0000 mgr.y (mgr.34107) 219 : cluster 0 pgmap v405: 105 pgs: 105 active+clean; 582 KiB data, 415 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 8.2 KiB/s wr, 87 op/s 2024-03-07T03:19:57.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:57 smithi145 bash[82329]: cluster 2024-03-07T03:19:57.329459+0000 mon.a (mon.0) 702 : cluster 0 osdmap e538: 8 total, 8 up, 8 in 2024-03-07T03:19:57.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:57 smithi189 bash[66969]: cluster 2024-03-07T03:19:56.410036+0000 mon.a (mon.0) 701 : cluster 0 osdmap e537: 8 total, 8 up, 8 in 2024-03-07T03:19:57.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:57 smithi189 bash[66969]: cluster 2024-03-07T03:19:56.634620+0000 mgr.y (mgr.34107) 219 : cluster 0 pgmap v405: 105 pgs: 105 active+clean; 582 KiB data, 415 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 8.2 KiB/s wr, 87 op/s 2024-03-07T03:19:57.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:57 smithi189 bash[66969]: cluster 2024-03-07T03:19:57.329459+0000 mon.a (mon.0) 702 : cluster 0 osdmap e538: 8 total, 8 up, 8 in 2024-03-07T03:19:59.465 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.SnapSetReleasesLock (2107 ms) 2024-03-07T03:19:59.465 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.FlattenLocksImage 2024-03-07T03:19:59.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:59 smithi145 bash[79878]: cluster 2024-03-07T03:19:58.430005+0000 mon.a (mon.0) 703 : cluster 0 osdmap e539: 8 total, 8 up, 8 in 2024-03-07T03:19:59.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:19:59 smithi145 bash[79878]: cluster 2024-03-07T03:19:58.635853+0000 mgr.y (mgr.34107) 220 : cluster 0 pgmap v408: 105 pgs: 105 active+clean; 582 KiB data, 424 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 7.7 KiB/s wr, 117 op/s 2024-03-07T03:19:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:59 smithi145 bash[82329]: cluster 2024-03-07T03:19:58.430005+0000 mon.a (mon.0) 703 : cluster 0 osdmap e539: 8 total, 8 up, 8 in 2024-03-07T03:19:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:19:59 smithi145 bash[82329]: cluster 2024-03-07T03:19:58.635853+0000 mgr.y (mgr.34107) 220 : cluster 0 pgmap v408: 105 pgs: 105 active+clean; 582 KiB data, 424 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 7.7 KiB/s wr, 117 op/s 2024-03-07T03:19:59.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:59 smithi189 bash[66969]: cluster 2024-03-07T03:19:58.430005+0000 mon.a (mon.0) 703 : cluster 0 osdmap e539: 8 total, 8 up, 8 in 2024-03-07T03:19:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:19:59 smithi189 bash[66969]: cluster 2024-03-07T03:19:58.635853+0000 mgr.y (mgr.34107) 220 : cluster 0 pgmap v408: 105 pgs: 105 active+clean; 582 KiB data, 424 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 7.7 KiB/s wr, 117 op/s 2024-03-07T03:20:00.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:00 smithi145 bash[79878]: debug 2024-03-07T03:19:59.997+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:20:00.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:00 smithi145 bash[79878]: cluster 2024-03-07T03:19:59.442933+0000 mon.a (mon.0) 704 : cluster 0 osdmap e540: 8 total, 8 up, 8 in 2024-03-07T03:20:00.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:00 smithi145 bash[79878]: cluster 2024-03-07T03:20:00.000177+0000 mon.a (mon.0) 705 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:20:00.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:00 smithi145 bash[82329]: cluster 2024-03-07T03:19:59.442933+0000 mon.a (mon.0) 704 : cluster 0 osdmap e540: 8 total, 8 up, 8 in 2024-03-07T03:20:00.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:00 smithi145 bash[82329]: cluster 2024-03-07T03:20:00.000177+0000 mon.a (mon.0) 705 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:20:00.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:00 smithi189 bash[66969]: cluster 2024-03-07T03:19:59.442933+0000 mon.a (mon.0) 704 : cluster 0 osdmap e540: 8 total, 8 up, 8 in 2024-03-07T03:20:00.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:00 smithi189 bash[66969]: cluster 2024-03-07T03:20:00.000177+0000 mon.a (mon.0) 705 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:20:01.494 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.FlattenLocksImage (2029 ms) 2024-03-07T03:20:01.494 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.FlattenFailsToLockImage 2024-03-07T03:20:01.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:01 smithi145 bash[79878]: cluster 2024-03-07T03:20:00.443055+0000 mon.a (mon.0) 706 : cluster 0 osdmap e541: 8 total, 8 up, 8 in 2024-03-07T03:20:01.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:01 smithi145 bash[79878]: cluster 2024-03-07T03:20:00.636598+0000 mgr.y (mgr.34107) 221 : cluster 0 pgmap v411: 105 pgs: 105 active+clean; 582 KiB data, 424 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 6.0 KiB/s wr, 111 op/s 2024-03-07T03:20:01.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:01 smithi145 bash[82329]: cluster 2024-03-07T03:20:00.443055+0000 mon.a (mon.0) 706 : cluster 0 osdmap e541: 8 total, 8 up, 8 in 2024-03-07T03:20:01.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:01 smithi145 bash[82329]: cluster 2024-03-07T03:20:00.636598+0000 mgr.y (mgr.34107) 221 : cluster 0 pgmap v411: 105 pgs: 105 active+clean; 582 KiB data, 424 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 6.0 KiB/s wr, 111 op/s 2024-03-07T03:20:01.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:01 smithi189 bash[66969]: cluster 2024-03-07T03:20:00.443055+0000 mon.a (mon.0) 706 : cluster 0 osdmap e541: 8 total, 8 up, 8 in 2024-03-07T03:20:01.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:01 smithi189 bash[66969]: cluster 2024-03-07T03:20:00.636598+0000 mgr.y (mgr.34107) 221 : cluster 0 pgmap v411: 105 pgs: 105 active+clean; 582 KiB data, 424 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 6.0 KiB/s wr, 111 op/s 2024-03-07T03:20:02.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:20:02 smithi189 bash[59814]: ts=2024-03-07T03:20:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:02.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:02 smithi189 bash[66969]: cluster 2024-03-07T03:20:01.467795+0000 mon.a (mon.0) 707 : cluster 0 osdmap e542: 8 total, 8 up, 8 in 2024-03-07T03:20:02.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:02 smithi189 bash[66969]: cluster 2024-03-07T03:20:02.332417+0000 mon.a (mon.0) 708 : cluster 0 osdmap e543: 8 total, 8 up, 8 in 2024-03-07T03:20:02.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:02 smithi145 bash[79878]: cluster 2024-03-07T03:20:01.467795+0000 mon.a (mon.0) 707 : cluster 0 osdmap e542: 8 total, 8 up, 8 in 2024-03-07T03:20:02.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:02 smithi145 bash[79878]: cluster 2024-03-07T03:20:02.332417+0000 mon.a (mon.0) 708 : cluster 0 osdmap e543: 8 total, 8 up, 8 in 2024-03-07T03:20:02.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:02 smithi145 bash[82329]: cluster 2024-03-07T03:20:01.467795+0000 mon.a (mon.0) 707 : cluster 0 osdmap e542: 8 total, 8 up, 8 in 2024-03-07T03:20:02.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:02 smithi145 bash[82329]: cluster 2024-03-07T03:20:02.332417+0000 mon.a (mon.0) 708 : cluster 0 osdmap e543: 8 total, 8 up, 8 in 2024-03-07T03:20:03.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:20:02] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.43.0" 2024-03-07T03:20:03.509 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.FlattenFailsToLockImage (2015 ms) 2024-03-07T03:20:03.509 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklisted 2024-03-07T03:20:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:03 smithi145 bash[79878]: cluster 2024-03-07T03:20:02.637376+0000 mgr.y (mgr.34107) 222 : cluster 0 pgmap v414: 105 pgs: 105 active+clean; 582 KiB data, 428 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 1.7 KiB/s wr, 25 op/s 2024-03-07T03:20:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:03 smithi145 bash[79878]: audit 2024-03-07T03:20:02.639217+0000 mon.a (mon.0) 709 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:03.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:03 smithi145 bash[82329]: cluster 2024-03-07T03:20:02.637376+0000 mgr.y (mgr.34107) 222 : cluster 0 pgmap v414: 105 pgs: 105 active+clean; 582 KiB data, 428 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 1.7 KiB/s wr, 25 op/s 2024-03-07T03:20:03.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:03 smithi145 bash[82329]: audit 2024-03-07T03:20:02.639217+0000 mon.a (mon.0) 709 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:03.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:03 smithi189 bash[66969]: cluster 2024-03-07T03:20:02.637376+0000 mgr.y (mgr.34107) 222 : cluster 0 pgmap v414: 105 pgs: 105 active+clean; 582 KiB data, 428 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 1.7 KiB/s wr, 25 op/s 2024-03-07T03:20:03.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:03 smithi189 bash[66969]: audit 2024-03-07T03:20:02.639217+0000 mon.a (mon.0) 709 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:04.487 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklisted (978 ms) 2024-03-07T03:20:04.487 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklistedWatch 2024-03-07T03:20:04.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:04 smithi145 bash[82329]: cluster 2024-03-07T03:20:03.480247+0000 mon.a (mon.0) 710 : cluster 0 osdmap e544: 8 total, 8 up, 8 in 2024-03-07T03:20:04.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:04 smithi145 bash[82329]: audit 2024-03-07T03:20:03.606651+0000 mon.c (mon.2) 7 : audit 1 from='client.? 172.21.15.145:0/3252190224' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3252190224"}]: dispatch 2024-03-07T03:20:04.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:04 smithi145 bash[82329]: audit 2024-03-07T03:20:03.607051+0000 mon.a (mon.0) 711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3252190224"}]: dispatch 2024-03-07T03:20:04.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:04 smithi145 bash[82329]: audit 2024-03-07T03:20:03.616142+0000 mon.a (mon.0) 712 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:20:04.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:04 smithi145 bash[79878]: cluster 2024-03-07T03:20:03.480247+0000 mon.a (mon.0) 710 : cluster 0 osdmap e544: 8 total, 8 up, 8 in 2024-03-07T03:20:04.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:04 smithi145 bash[79878]: audit 2024-03-07T03:20:03.606651+0000 mon.c (mon.2) 7 : audit 1 from='client.? 172.21.15.145:0/3252190224' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3252190224"}]: dispatch 2024-03-07T03:20:04.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:04 smithi145 bash[79878]: audit 2024-03-07T03:20:03.607051+0000 mon.a (mon.0) 711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3252190224"}]: dispatch 2024-03-07T03:20:04.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:04 smithi145 bash[79878]: audit 2024-03-07T03:20:03.616142+0000 mon.a (mon.0) 712 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:20:04.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:04 smithi189 bash[66969]: cluster 2024-03-07T03:20:03.480247+0000 mon.a (mon.0) 710 : cluster 0 osdmap e544: 8 total, 8 up, 8 in 2024-03-07T03:20:04.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:04 smithi189 bash[66969]: audit 2024-03-07T03:20:03.606651+0000 mon.c (mon.2) 7 : audit 1 from='client.? 172.21.15.145:0/3252190224' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3252190224"}]: dispatch 2024-03-07T03:20:04.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:04 smithi189 bash[66969]: audit 2024-03-07T03:20:03.607051+0000 mon.a (mon.0) 711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3252190224"}]: dispatch 2024-03-07T03:20:04.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:04 smithi189 bash[66969]: audit 2024-03-07T03:20:03.616142+0000 mon.a (mon.0) 712 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:20:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:05 smithi145 bash[79878]: audit 2024-03-07T03:20:04.472843+0000 mon.a (mon.0) 713 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3252190224"}]': finished 2024-03-07T03:20:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:05 smithi145 bash[79878]: cluster 2024-03-07T03:20:04.478650+0000 mon.a (mon.0) 714 : cluster 0 osdmap e545: 8 total, 8 up, 8 in 2024-03-07T03:20:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:05 smithi145 bash[79878]: audit 2024-03-07T03:20:04.550407+0000 mon.c (mon.2) 8 : audit 1 from='client.? 172.21.15.145:0/3395344828' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3395344828"}]: dispatch 2024-03-07T03:20:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:05 smithi145 bash[79878]: audit 2024-03-07T03:20:04.550774+0000 mon.a (mon.0) 715 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3395344828"}]: dispatch 2024-03-07T03:20:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:05 smithi145 bash[79878]: cluster 2024-03-07T03:20:04.638163+0000 mgr.y (mgr.34107) 223 : cluster 0 pgmap v417: 105 pgs: 105 active+clean; 583 KiB data, 433 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 9.5 KiB/s wr, 179 op/s 2024-03-07T03:20:05.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:05 smithi145 bash[82329]: audit 2024-03-07T03:20:04.472843+0000 mon.a (mon.0) 713 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3252190224"}]': finished 2024-03-07T03:20:05.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:05 smithi145 bash[82329]: cluster 2024-03-07T03:20:04.478650+0000 mon.a (mon.0) 714 : cluster 0 osdmap e545: 8 total, 8 up, 8 in 2024-03-07T03:20:05.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:05 smithi145 bash[82329]: audit 2024-03-07T03:20:04.550407+0000 mon.c (mon.2) 8 : audit 1 from='client.? 172.21.15.145:0/3395344828' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3395344828"}]: dispatch 2024-03-07T03:20:05.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:05 smithi145 bash[82329]: audit 2024-03-07T03:20:04.550774+0000 mon.a (mon.0) 715 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3395344828"}]: dispatch 2024-03-07T03:20:05.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:05 smithi145 bash[82329]: cluster 2024-03-07T03:20:04.638163+0000 mgr.y (mgr.34107) 223 : cluster 0 pgmap v417: 105 pgs: 105 active+clean; 583 KiB data, 433 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 9.5 KiB/s wr, 179 op/s 2024-03-07T03:20:05.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:05 smithi189 bash[66969]: audit 2024-03-07T03:20:04.472843+0000 mon.a (mon.0) 713 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3252190224"}]': finished 2024-03-07T03:20:05.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:05 smithi189 bash[66969]: cluster 2024-03-07T03:20:04.478650+0000 mon.a (mon.0) 714 : cluster 0 osdmap e545: 8 total, 8 up, 8 in 2024-03-07T03:20:05.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:05 smithi189 bash[66969]: audit 2024-03-07T03:20:04.550407+0000 mon.c (mon.2) 8 : audit 1 from='client.? 172.21.15.145:0/3395344828' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3395344828"}]: dispatch 2024-03-07T03:20:05.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:05 smithi189 bash[66969]: audit 2024-03-07T03:20:04.550774+0000 mon.a (mon.0) 715 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3395344828"}]: dispatch 2024-03-07T03:20:05.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:05 smithi189 bash[66969]: cluster 2024-03-07T03:20:04.638163+0000 mgr.y (mgr.34107) 223 : cluster 0 pgmap v417: 105 pgs: 105 active+clean; 583 KiB data, 433 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 9.5 KiB/s wr, 179 op/s 2024-03-07T03:20:06.485 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklistedWatch (1998 ms) 2024-03-07T03:20:06.485 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.AioWriteRequestsLock 2024-03-07T03:20:06.553 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.AioWriteRequestsLock (68 ms) 2024-03-07T03:20:06.553 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.AioDiscardRequestsLock 2024-03-07T03:20:06.620 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.AioDiscardRequestsLock (67 ms) 2024-03-07T03:20:06.620 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.CancelAsyncResize 2024-03-07T03:20:06.724 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.CancelAsyncResize (104 ms) 2024-03-07T03:20:06.724 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.MultipleResize 2024-03-07T03:20:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:06 smithi145 bash[79878]: audit 2024-03-07T03:20:05.484048+0000 mon.a (mon.0) 716 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3395344828"}]': finished 2024-03-07T03:20:06.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:06 smithi145 bash[79878]: cluster 2024-03-07T03:20:05.490580+0000 mon.a (mon.0) 717 : cluster 0 osdmap e546: 8 total, 8 up, 8 in 2024-03-07T03:20:06.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:06 smithi145 bash[82329]: audit 2024-03-07T03:20:05.484048+0000 mon.a (mon.0) 716 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3395344828"}]': finished 2024-03-07T03:20:06.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:06 smithi145 bash[82329]: cluster 2024-03-07T03:20:05.490580+0000 mon.a (mon.0) 717 : cluster 0 osdmap e546: 8 total, 8 up, 8 in 2024-03-07T03:20:06.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:06 smithi189 bash[66969]: audit 2024-03-07T03:20:05.484048+0000 mon.a (mon.0) 716 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/3395344828"}]': finished 2024-03-07T03:20:06.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:06 smithi189 bash[66969]: cluster 2024-03-07T03:20:05.490580+0000 mon.a (mon.0) 717 : cluster 0 osdmap e546: 8 total, 8 up, 8 in 2024-03-07T03:20:06.875 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.MultipleResize (151 ms) 2024-03-07T03:20:06.875 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.Metadata 2024-03-07T03:20:07.015 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.Metadata (140 ms) 2024-03-07T03:20:07.015 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.MetadataConfApply 2024-03-07T03:20:07.098 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.MetadataConfApply (83 ms) 2024-03-07T03:20:07.098 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.SnapshotCopyup 2024-03-07T03:20:07.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:07 smithi145 bash[82329]: cluster 2024-03-07T03:20:06.638868+0000 mgr.y (mgr.34107) 224 : cluster 0 pgmap v419: 105 pgs: 105 active+clean; 583 KiB data, 433 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 8.3 KiB/s wr, 149 op/s 2024-03-07T03:20:07.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:07 smithi145 bash[82329]: cluster 2024-03-07T03:20:07.332613+0000 mon.a (mon.0) 718 : cluster 0 osdmap e547: 8 total, 8 up, 8 in 2024-03-07T03:20:07.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:07 smithi145 bash[79878]: cluster 2024-03-07T03:20:06.638868+0000 mgr.y (mgr.34107) 224 : cluster 0 pgmap v419: 105 pgs: 105 active+clean; 583 KiB data, 433 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 8.3 KiB/s wr, 149 op/s 2024-03-07T03:20:07.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:07 smithi145 bash[79878]: cluster 2024-03-07T03:20:07.332613+0000 mon.a (mon.0) 718 : cluster 0 osdmap e547: 8 total, 8 up, 8 in 2024-03-07T03:20:07.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:07 smithi189 bash[66969]: cluster 2024-03-07T03:20:06.638868+0000 mgr.y (mgr.34107) 224 : cluster 0 pgmap v419: 105 pgs: 105 active+clean; 583 KiB data, 433 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 8.3 KiB/s wr, 149 op/s 2024-03-07T03:20:07.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:07 smithi189 bash[66969]: cluster 2024-03-07T03:20:07.332613+0000 mon.a (mon.0) 718 : cluster 0 osdmap e547: 8 total, 8 up, 8 in 2024-03-07T03:20:09.699 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.SnapshotCopyup (2601 ms) 2024-03-07T03:20:09.699 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.SnapshotCopyupZeros 2024-03-07T03:20:09.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:09 smithi145 bash[79878]: cluster 2024-03-07T03:20:08.510246+0000 mon.a (mon.0) 719 : cluster 0 osdmap e548: 8 total, 8 up, 8 in 2024-03-07T03:20:09.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:09 smithi145 bash[79878]: cluster 2024-03-07T03:20:08.640007+0000 mgr.y (mgr.34107) 225 : cluster 0 pgmap v422: 105 pgs: 105 active+clean; 585 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 173 KiB/s rd, 24 KiB/s wr, 249 op/s 2024-03-07T03:20:09.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:09 smithi145 bash[82329]: cluster 2024-03-07T03:20:08.510246+0000 mon.a (mon.0) 719 : cluster 0 osdmap e548: 8 total, 8 up, 8 in 2024-03-07T03:20:09.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:09 smithi145 bash[82329]: cluster 2024-03-07T03:20:08.640007+0000 mgr.y (mgr.34107) 225 : cluster 0 pgmap v422: 105 pgs: 105 active+clean; 585 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 173 KiB/s rd, 24 KiB/s wr, 249 op/s 2024-03-07T03:20:09.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:09 smithi189 bash[66969]: cluster 2024-03-07T03:20:08.510246+0000 mon.a (mon.0) 719 : cluster 0 osdmap e548: 8 total, 8 up, 8 in 2024-03-07T03:20:09.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:09 smithi189 bash[66969]: cluster 2024-03-07T03:20:08.640007+0000 mgr.y (mgr.34107) 225 : cluster 0 pgmap v422: 105 pgs: 105 active+clean; 585 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 173 KiB/s rd, 24 KiB/s wr, 249 op/s 2024-03-07T03:20:10.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:10 smithi145 bash[79878]: cluster 2024-03-07T03:20:09.519355+0000 mon.a (mon.0) 720 : cluster 0 osdmap e549: 8 total, 8 up, 8 in 2024-03-07T03:20:10.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:10 smithi145 bash[82329]: cluster 2024-03-07T03:20:09.519355+0000 mon.a (mon.0) 720 : cluster 0 osdmap e549: 8 total, 8 up, 8 in 2024-03-07T03:20:10.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:10 smithi189 bash[66969]: cluster 2024-03-07T03:20:09.519355+0000 mon.a (mon.0) 720 : cluster 0 osdmap e549: 8 total, 8 up, 8 in 2024-03-07T03:20:11.602 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.SnapshotCopyupZeros (1903 ms) 2024-03-07T03:20:11.602 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.SnapshotCopyupZerosMigration 2024-03-07T03:20:11.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:11 smithi145 bash[82329]: cluster 2024-03-07T03:20:10.525372+0000 mon.a (mon.0) 721 : cluster 0 osdmap e550: 8 total, 8 up, 8 in 2024-03-07T03:20:11.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:11 smithi145 bash[82329]: cluster 2024-03-07T03:20:10.640769+0000 mgr.y (mgr.34107) 226 : cluster 0 pgmap v425: 105 pgs: 105 active+clean; 585 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 175 KiB/s rd, 23 KiB/s wr, 253 op/s 2024-03-07T03:20:11.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:11 smithi145 bash[79878]: cluster 2024-03-07T03:20:10.525372+0000 mon.a (mon.0) 721 : cluster 0 osdmap e550: 8 total, 8 up, 8 in 2024-03-07T03:20:11.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:11 smithi145 bash[79878]: cluster 2024-03-07T03:20:10.640769+0000 mgr.y (mgr.34107) 226 : cluster 0 pgmap v425: 105 pgs: 105 active+clean; 585 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 175 KiB/s rd, 23 KiB/s wr, 253 op/s 2024-03-07T03:20:11.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:11 smithi189 bash[66969]: cluster 2024-03-07T03:20:10.525372+0000 mon.a (mon.0) 721 : cluster 0 osdmap e550: 8 total, 8 up, 8 in 2024-03-07T03:20:11.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:11 smithi189 bash[66969]: cluster 2024-03-07T03:20:10.640769+0000 mgr.y (mgr.34107) 226 : cluster 0 pgmap v425: 105 pgs: 105 active+clean; 585 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 175 KiB/s rd, 23 KiB/s wr, 253 op/s 2024-03-07T03:20:12.383 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.SnapshotCopyupZerosMigration (781 ms) 2024-03-07T03:20:12.383 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.ResizeCopyup 2024-03-07T03:20:12.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:12 smithi189 bash[66969]: cluster 2024-03-07T03:20:11.550711+0000 mon.a (mon.0) 722 : cluster 0 osdmap e551: 8 total, 8 up, 8 in 2024-03-07T03:20:12.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:12 smithi189 bash[66969]: cluster 2024-03-07T03:20:12.338405+0000 mon.a (mon.0) 723 : cluster 0 osdmap e552: 8 total, 8 up, 8 in 2024-03-07T03:20:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:20:12 smithi189 bash[59814]: ts=2024-03-07T03:20:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:12.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:12 smithi145 bash[79878]: cluster 2024-03-07T03:20:11.550711+0000 mon.a (mon.0) 722 : cluster 0 osdmap e551: 8 total, 8 up, 8 in 2024-03-07T03:20:12.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:12 smithi145 bash[79878]: cluster 2024-03-07T03:20:12.338405+0000 mon.a (mon.0) 723 : cluster 0 osdmap e552: 8 total, 8 up, 8 in 2024-03-07T03:20:12.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:20:12] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.43.0" 2024-03-07T03:20:12.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:12 smithi145 bash[82329]: cluster 2024-03-07T03:20:11.550711+0000 mon.a (mon.0) 722 : cluster 0 osdmap e551: 8 total, 8 up, 8 in 2024-03-07T03:20:12.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:12 smithi145 bash[82329]: cluster 2024-03-07T03:20:12.338405+0000 mon.a (mon.0) 723 : cluster 0 osdmap e552: 8 total, 8 up, 8 in 2024-03-07T03:20:13.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:13 smithi145 bash[79878]: cluster 2024-03-07T03:20:12.641429+0000 mgr.y (mgr.34107) 227 : cluster 0 pgmap v428: 105 pgs: 105 active+clean; 586 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 8.7 KiB/s wr, 176 op/s 2024-03-07T03:20:13.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:13 smithi145 bash[79878]: cluster 2024-03-07T03:20:13.336362+0000 mon.a (mon.0) 724 : cluster 0 osdmap e553: 8 total, 8 up, 8 in 2024-03-07T03:20:13.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:13 smithi145 bash[82329]: cluster 2024-03-07T03:20:12.641429+0000 mgr.y (mgr.34107) 227 : cluster 0 pgmap v428: 105 pgs: 105 active+clean; 586 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 8.7 KiB/s wr, 176 op/s 2024-03-07T03:20:13.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:13 smithi145 bash[82329]: cluster 2024-03-07T03:20:13.336362+0000 mon.a (mon.0) 724 : cluster 0 osdmap e553: 8 total, 8 up, 8 in 2024-03-07T03:20:13.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:13 smithi189 bash[66969]: cluster 2024-03-07T03:20:12.641429+0000 mgr.y (mgr.34107) 227 : cluster 0 pgmap v428: 105 pgs: 105 active+clean; 586 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 8.7 KiB/s wr, 176 op/s 2024-03-07T03:20:13.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:13 smithi189 bash[66969]: cluster 2024-03-07T03:20:13.336362+0000 mon.a (mon.0) 724 : cluster 0 osdmap e553: 8 total, 8 up, 8 in 2024-03-07T03:20:14.452 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.ResizeCopyup (2069 ms) 2024-03-07T03:20:14.452 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.DiscardCopyup 2024-03-07T03:20:14.473 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_internal.cc:1056: Skipped 2024-03-07T03:20:14.473 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:20:14.473 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestInternal.DiscardCopyup (21 ms) 2024-03-07T03:20:14.473 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.ImageOptions 2024-03-07T03:20:14.494 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.ImageOptions (21 ms) 2024-03-07T03:20:14.494 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.WriteFullCopyup 2024-03-07T03:20:15.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:15 smithi145 bash[79878]: cluster 2024-03-07T03:20:14.340116+0000 mon.a (mon.0) 725 : cluster 0 osdmap e554: 8 total, 8 up, 8 in 2024-03-07T03:20:15.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:15 smithi145 bash[79878]: cluster 2024-03-07T03:20:14.642116+0000 mgr.y (mgr.34107) 228 : cluster 0 pgmap v431: 105 pgs: 105 active+clean; 596 KiB data, 422 MiB used, 715 GiB / 715 GiB avail; 261 KiB/s rd, 24 KiB/s wr, 358 op/s 2024-03-07T03:20:15.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:15 smithi145 bash[82329]: cluster 2024-03-07T03:20:14.340116+0000 mon.a (mon.0) 725 : cluster 0 osdmap e554: 8 total, 8 up, 8 in 2024-03-07T03:20:15.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:15 smithi145 bash[82329]: cluster 2024-03-07T03:20:14.642116+0000 mgr.y (mgr.34107) 228 : cluster 0 pgmap v431: 105 pgs: 105 active+clean; 596 KiB data, 422 MiB used, 715 GiB / 715 GiB avail; 261 KiB/s rd, 24 KiB/s wr, 358 op/s 2024-03-07T03:20:15.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:15 smithi189 bash[66969]: cluster 2024-03-07T03:20:14.340116+0000 mon.a (mon.0) 725 : cluster 0 osdmap e554: 8 total, 8 up, 8 in 2024-03-07T03:20:15.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:15 smithi189 bash[66969]: cluster 2024-03-07T03:20:14.642116+0000 mgr.y (mgr.34107) 228 : cluster 0 pgmap v431: 105 pgs: 105 active+clean; 596 KiB data, 422 MiB used, 715 GiB / 715 GiB avail; 261 KiB/s rd, 24 KiB/s wr, 358 op/s 2024-03-07T03:20:16.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:16 smithi145 bash[79878]: cluster 2024-03-07T03:20:15.356749+0000 mon.a (mon.0) 726 : cluster 0 osdmap e555: 8 total, 8 up, 8 in 2024-03-07T03:20:16.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:16 smithi145 bash[82329]: cluster 2024-03-07T03:20:15.356749+0000 mon.a (mon.0) 726 : cluster 0 osdmap e555: 8 total, 8 up, 8 in 2024-03-07T03:20:16.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:16 smithi189 bash[66969]: cluster 2024-03-07T03:20:15.356749+0000 mon.a (mon.0) 726 : cluster 0 osdmap e555: 8 total, 8 up, 8 in 2024-03-07T03:20:17.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:17 smithi145 bash[79878]: cluster 2024-03-07T03:20:16.353634+0000 mon.a (mon.0) 727 : cluster 0 osdmap e556: 8 total, 8 up, 8 in 2024-03-07T03:20:17.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:17 smithi145 bash[79878]: cluster 2024-03-07T03:20:16.642982+0000 mgr.y (mgr.34107) 229 : cluster 0 pgmap v434: 105 pgs: 105 active+clean; 604 KiB data, 426 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 20 KiB/s wr, 195 op/s 2024-03-07T03:20:17.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:17 smithi145 bash[82329]: cluster 2024-03-07T03:20:16.353634+0000 mon.a (mon.0) 727 : cluster 0 osdmap e556: 8 total, 8 up, 8 in 2024-03-07T03:20:17.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:17 smithi145 bash[82329]: cluster 2024-03-07T03:20:16.642982+0000 mgr.y (mgr.34107) 229 : cluster 0 pgmap v434: 105 pgs: 105 active+clean; 604 KiB data, 426 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 20 KiB/s wr, 195 op/s 2024-03-07T03:20:17.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:17 smithi189 bash[66969]: cluster 2024-03-07T03:20:16.353634+0000 mon.a (mon.0) 727 : cluster 0 osdmap e556: 8 total, 8 up, 8 in 2024-03-07T03:20:17.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:17 smithi189 bash[66969]: cluster 2024-03-07T03:20:16.642982+0000 mgr.y (mgr.34107) 229 : cluster 0 pgmap v434: 105 pgs: 105 active+clean; 604 KiB data, 426 MiB used, 715 GiB / 715 GiB avail; 130 KiB/s rd, 20 KiB/s wr, 195 op/s 2024-03-07T03:20:18.402 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.WriteFullCopyup (3908 ms) 2024-03-07T03:20:18.403 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.DiffIterateCloneOverwrite 2024-03-07T03:20:18.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:18 smithi145 bash[79878]: cluster 2024-03-07T03:20:17.372729+0000 mon.a (mon.0) 728 : cluster 0 osdmap e557: 8 total, 8 up, 8 in 2024-03-07T03:20:18.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:18 smithi145 bash[79878]: audit 2024-03-07T03:20:17.639198+0000 mon.a (mon.0) 729 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:18.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:18 smithi145 bash[82329]: cluster 2024-03-07T03:20:17.372729+0000 mon.a (mon.0) 728 : cluster 0 osdmap e557: 8 total, 8 up, 8 in 2024-03-07T03:20:18.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:18 smithi145 bash[82329]: audit 2024-03-07T03:20:17.639198+0000 mon.a (mon.0) 729 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:18.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:18 smithi189 bash[66969]: cluster 2024-03-07T03:20:17.372729+0000 mon.a (mon.0) 728 : cluster 0 osdmap e557: 8 total, 8 up, 8 in 2024-03-07T03:20:18.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:18 smithi189 bash[66969]: audit 2024-03-07T03:20:17.639198+0000 mon.a (mon.0) 729 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:19.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:19 smithi145 bash[79878]: cluster 2024-03-07T03:20:18.383086+0000 mon.a (mon.0) 730 : cluster 0 osdmap e558: 8 total, 8 up, 8 in 2024-03-07T03:20:19.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:19 smithi145 bash[79878]: cluster 2024-03-07T03:20:18.644172+0000 mgr.y (mgr.34107) 230 : cluster 0 pgmap v437: 105 pgs: 105 active+clean; 4.6 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 277 op/s 2024-03-07T03:20:19.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:19 smithi145 bash[82329]: cluster 2024-03-07T03:20:18.383086+0000 mon.a (mon.0) 730 : cluster 0 osdmap e558: 8 total, 8 up, 8 in 2024-03-07T03:20:19.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:19 smithi145 bash[82329]: cluster 2024-03-07T03:20:18.644172+0000 mgr.y (mgr.34107) 230 : cluster 0 pgmap v437: 105 pgs: 105 active+clean; 4.6 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 277 op/s 2024-03-07T03:20:19.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:19 smithi189 bash[66969]: cluster 2024-03-07T03:20:18.383086+0000 mon.a (mon.0) 730 : cluster 0 osdmap e558: 8 total, 8 up, 8 in 2024-03-07T03:20:19.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:19 smithi189 bash[66969]: cluster 2024-03-07T03:20:18.644172+0000 mgr.y (mgr.34107) 230 : cluster 0 pgmap v437: 105 pgs: 105 active+clean; 4.6 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 277 op/s 2024-03-07T03:20:20.436 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.DiffIterateCloneOverwrite (2034 ms) 2024-03-07T03:20:20.436 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.TestCoR 2024-03-07T03:20:20.458 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_internal.cc:1313: Skipped 2024-03-07T03:20:20.459 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to disabled copy-on-read 2024-03-07T03:20:20.459 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestInternal.TestCoR (23 ms) 2024-03-07T03:20:20.459 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.FlattenNoEmptyObjects 2024-03-07T03:20:20.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:20 smithi145 bash[79878]: cluster 2024-03-07T03:20:19.392701+0000 mon.a (mon.0) 731 : cluster 0 osdmap e559: 8 total, 8 up, 8 in 2024-03-07T03:20:20.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:20 smithi145 bash[82329]: cluster 2024-03-07T03:20:19.392701+0000 mon.a (mon.0) 731 : cluster 0 osdmap e559: 8 total, 8 up, 8 in 2024-03-07T03:20:20.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:20 smithi189 bash[66969]: cluster 2024-03-07T03:20:19.392701+0000 mon.a (mon.0) 731 : cluster 0 osdmap e559: 8 total, 8 up, 8 in 2024-03-07T03:20:21.067 INFO:tasks.workunit.client.0.smithi145.stdout:made parent image "image396": 4194304K (1024 * 4K) 2024-03-07T03:20:21.067 INFO:tasks.workunit.client.0.smithi145.stdout:generated random write map: 2024-03-07T03:20:21.067 INFO:tasks.workunit.client.0.smithi145.stdout: [25 , 103615 ] 2024-03-07T03:20:21.067 INFO:tasks.workunit.client.0.smithi145.stdout: [29 , 119708 ] 2024-03-07T03:20:21.067 INFO:tasks.workunit.client.0.smithi145.stdout: [53 , 218312 ] 2024-03-07T03:20:21.067 INFO:tasks.workunit.client.0.smithi145.stdout: [56 , 230871 ] 2024-03-07T03:20:21.067 INFO:tasks.workunit.client.0.smithi145.stdout: [66 , 272871 ] 2024-03-07T03:20:21.067 INFO:tasks.workunit.client.0.smithi145.stdout: [81 , 333728 ] 2024-03-07T03:20:21.067 INFO:tasks.workunit.client.0.smithi145.stdout: [88 , 363325 ] 2024-03-07T03:20:21.067 INFO:tasks.workunit.client.0.smithi145.stdout: [91 , 375880 ] 2024-03-07T03:20:21.067 INFO:tasks.workunit.client.0.smithi145.stdout: [101 , 414475 ] 2024-03-07T03:20:21.068 INFO:tasks.workunit.client.0.smithi145.stdout: [105 , 432176 ] 2024-03-07T03:20:21.068 INFO:tasks.workunit.client.0.smithi145.stdout: [115 , 473330 ] 2024-03-07T03:20:21.068 INFO:tasks.workunit.client.0.smithi145.stdout: [122 , 502070 ] 2024-03-07T03:20:21.068 INFO:tasks.workunit.client.0.smithi145.stdout: [133 , 547127 ] 2024-03-07T03:20:21.068 INFO:tasks.workunit.client.0.smithi145.stdout: [140 , 574875 ] 2024-03-07T03:20:21.068 INFO:tasks.workunit.client.0.smithi145.stdout: [167 , 686897 ] 2024-03-07T03:20:21.068 INFO:tasks.workunit.client.0.smithi145.stdout: [168 , 691542 ] 2024-03-07T03:20:21.068 INFO:tasks.workunit.client.0.smithi145.stdout: [180 , 739888 ] 2024-03-07T03:20:21.068 INFO:tasks.workunit.client.0.smithi145.stdout: [189 , 774923 ] 2024-03-07T03:20:21.068 INFO:tasks.workunit.client.0.smithi145.stdout: [215 , 881765 ] 2024-03-07T03:20:21.068 INFO:tasks.workunit.client.0.smithi145.stdout: [236 , 968904 ] 2024-03-07T03:20:21.068 INFO:tasks.workunit.client.0.smithi145.stdout: [240 , 985930 ] 2024-03-07T03:20:21.068 INFO:tasks.workunit.client.0.smithi145.stdout: [243 , 996487 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [244 , 999830 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [259 , 1062454 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [262 , 1074613 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [269 , 1105322 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [270 , 1108983 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [277 , 1135444 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [285 , 1170770 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [286 , 1172472 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [289 , 1186985 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [301 , 1234933 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [321 , 1315682 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [323 , 1323142 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [347 , 1421737 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [356 , 1461105 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [360 , 1477277 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [373 , 1530712 ] 2024-03-07T03:20:21.069 INFO:tasks.workunit.client.0.smithi145.stdout: [384 , 1575869 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [387 , 1588659 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [399 , 1637290 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [402 , 1649714 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [415 , 1699899 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [434 , 1781105 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [458 , 1877657 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [460 , 1886736 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [473 , 1938270 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [480 , 1969599 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [489 , 2003382 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [491 , 2012803 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [517 , 2119604 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [526 , 2157698 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [533 , 2186712 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [549 , 2250095 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [562 , 2303234 ] 2024-03-07T03:20:21.070 INFO:tasks.workunit.client.0.smithi145.stdout: [577 , 2366507 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [591 , 2420972 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [602 , 2468439 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [625 , 2562437 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [629 , 2579286 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [634 , 2599325 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [637 , 2610107 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [640 , 2623639 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [652 , 2670620 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [658 , 2695844 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [669 , 2742271 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [672 , 2753949 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [675 , 2766609 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [683 , 2798102 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [684 , 2804863 ] 2024-03-07T03:20:21.071 INFO:tasks.workunit.client.0.smithi145.stdout: [723 , 2962491 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [748 , 3065670 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [751 , 3076792 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [753 , 3086605 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [754 , 3090111 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [756 , 3097097 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [762 , 3122574 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [796 , 3262258 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [801 , 3282260 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [802 , 3285231 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [814 , 3337328 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [835 , 3423046 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [855 , 3503507 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [857 , 3513190 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [866 , 3549981 ] 2024-03-07T03:20:21.072 INFO:tasks.workunit.client.0.smithi145.stdout: [867 , 3552031 ] 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout: [873 , 3577777 ] 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout: [891 , 3649958 ] 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout: [893 , 3658085 ] 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout: [915 , 3750513 ] 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout: [929 , 3808672 ] 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout: [931 , 3813825 ] 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout: [939 , 3846451 ] 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout: [941 , 3857078 ] 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout: [947 , 3881811 ] 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout: [949 , 3889537 ] 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout: [1023 , 4193764 ] 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout:write data based on random map 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout: write object-25 2024-03-07T03:20:21.073 INFO:tasks.workunit.client.0.smithi145.stdout: write object-29 2024-03-07T03:20:21.074 INFO:tasks.workunit.client.0.smithi145.stdout: write object-53 2024-03-07T03:20:21.074 INFO:tasks.workunit.client.0.smithi145.stdout: write object-56 2024-03-07T03:20:21.074 INFO:tasks.workunit.client.0.smithi145.stdout: write object-66 2024-03-07T03:20:21.074 INFO:tasks.workunit.client.0.smithi145.stdout: write object-81 2024-03-07T03:20:21.074 INFO:tasks.workunit.client.0.smithi145.stdout: write object-88 2024-03-07T03:20:21.074 INFO:tasks.workunit.client.0.smithi145.stdout: write object-91 2024-03-07T03:20:21.074 INFO:tasks.workunit.client.0.smithi145.stdout: write object-101 2024-03-07T03:20:21.074 INFO:tasks.workunit.client.0.smithi145.stdout: write object-105 2024-03-07T03:20:21.074 INFO:tasks.workunit.client.0.smithi145.stdout: write object-115 2024-03-07T03:20:21.074 INFO:tasks.workunit.client.0.smithi145.stdout: write object-122 2024-03-07T03:20:21.074 INFO:tasks.workunit.client.0.smithi145.stdout: write object-133 2024-03-07T03:20:21.074 INFO:tasks.workunit.client.0.smithi145.stdout: write object-140 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-167 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-168 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-180 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-189 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-215 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-236 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-240 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-243 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-244 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-259 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-262 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-269 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-270 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-277 2024-03-07T03:20:21.075 INFO:tasks.workunit.client.0.smithi145.stdout: write object-285 2024-03-07T03:20:21.076 INFO:tasks.workunit.client.0.smithi145.stdout: write object-286 2024-03-07T03:20:21.076 INFO:tasks.workunit.client.0.smithi145.stdout: write object-289 2024-03-07T03:20:21.076 INFO:tasks.workunit.client.0.smithi145.stdout: write object-301 2024-03-07T03:20:21.076 INFO:tasks.workunit.client.0.smithi145.stdout: write object-321 2024-03-07T03:20:21.076 INFO:tasks.workunit.client.0.smithi145.stdout: write object-323 2024-03-07T03:20:21.076 INFO:tasks.workunit.client.0.smithi145.stdout: write object-347 2024-03-07T03:20:21.076 INFO:tasks.workunit.client.0.smithi145.stdout: write object-356 2024-03-07T03:20:21.076 INFO:tasks.workunit.client.0.smithi145.stdout: write object-360 2024-03-07T03:20:21.076 INFO:tasks.workunit.client.0.smithi145.stdout: write object-373 2024-03-07T03:20:21.076 INFO:tasks.workunit.client.0.smithi145.stdout: write object-384 2024-03-07T03:20:21.076 INFO:tasks.workunit.client.0.smithi145.stdout: write object-387 2024-03-07T03:20:21.076 INFO:tasks.workunit.client.0.smithi145.stdout: write object-399 2024-03-07T03:20:21.076 INFO:tasks.workunit.client.0.smithi145.stdout: write object-402 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-415 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-434 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-458 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-460 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-473 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-480 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-489 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-491 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-517 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-526 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-533 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-549 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-562 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-577 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-591 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-602 2024-03-07T03:20:21.077 INFO:tasks.workunit.client.0.smithi145.stdout: write object-625 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-629 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-634 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-637 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-640 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-652 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-658 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-669 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-672 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-675 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-683 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-684 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-723 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-748 2024-03-07T03:20:21.078 INFO:tasks.workunit.client.0.smithi145.stdout: write object-751 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-753 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-754 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-756 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-762 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-796 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-801 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-802 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-814 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-835 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-855 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-857 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-866 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-867 2024-03-07T03:20:21.079 INFO:tasks.workunit.client.0.smithi145.stdout: write object-873 2024-03-07T03:20:21.789 INFO:tasks.workunit.client.0.smithi145.stdout: write object-891 2024-03-07T03:20:21.789 INFO:tasks.workunit.client.0.smithi145.stdout: write object-893 2024-03-07T03:20:21.789 INFO:tasks.workunit.client.0.smithi145.stdout: write object-915 2024-03-07T03:20:21.790 INFO:tasks.workunit.client.0.smithi145.stdout: write object-929 2024-03-07T03:20:21.790 INFO:tasks.workunit.client.0.smithi145.stdout: write object-931 2024-03-07T03:20:21.790 INFO:tasks.workunit.client.0.smithi145.stdout: write object-939 2024-03-07T03:20:21.790 INFO:tasks.workunit.client.0.smithi145.stdout: write object-941 2024-03-07T03:20:21.790 INFO:tasks.workunit.client.0.smithi145.stdout: write object-947 2024-03-07T03:20:21.790 INFO:tasks.workunit.client.0.smithi145.stdout: write object-949 2024-03-07T03:20:21.791 INFO:tasks.workunit.client.0.smithi145.stdout: write object-1023 2024-03-07T03:20:21.791 INFO:tasks.workunit.client.0.smithi145.stdout:verify written data by reading 2024-03-07T03:20:21.791 INFO:tasks.workunit.client.0.smithi145.stdout: read object-25 2024-03-07T03:20:21.791 INFO:tasks.workunit.client.0.smithi145.stdout:made snapshot "image396@parent_snap" and protect it 2024-03-07T03:20:21.791 INFO:tasks.workunit.client.0.smithi145.stdout:made and opened clone "image397" 2024-03-07T03:20:21.792 INFO:tasks.workunit.client.0.smithi145.stdout:flattening clone: "image397" 2024-03-07T03:20:21.792 INFO:tasks.workunit.client.0.smithi145.stdout:check whether child image has the same set of objects as parent 2024-03-07T03:20:21.792 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.FlattenNoEmptyObjects (1330 ms) 2024-03-07T03:20:21.792 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.PoolMetadataConfApply 2024-03-07T03:20:21.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:21 smithi145 bash[79878]: cluster 2024-03-07T03:20:20.394068+0000 mon.a (mon.0) 732 : cluster 0 osdmap e560: 8 total, 8 up, 8 in 2024-03-07T03:20:21.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:21 smithi145 bash[79878]: cluster 2024-03-07T03:20:20.644813+0000 mgr.y (mgr.34107) 231 : cluster 0 pgmap v440: 105 pgs: 105 active+clean; 4.6 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 263 op/s 2024-03-07T03:20:21.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:21 smithi145 bash[82329]: cluster 2024-03-07T03:20:20.394068+0000 mon.a (mon.0) 732 : cluster 0 osdmap e560: 8 total, 8 up, 8 in 2024-03-07T03:20:21.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:21 smithi145 bash[82329]: cluster 2024-03-07T03:20:20.644813+0000 mgr.y (mgr.34107) 231 : cluster 0 pgmap v440: 105 pgs: 105 active+clean; 4.6 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 263 op/s 2024-03-07T03:20:21.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:21 smithi189 bash[66969]: cluster 2024-03-07T03:20:20.394068+0000 mon.a (mon.0) 732 : cluster 0 osdmap e560: 8 total, 8 up, 8 in 2024-03-07T03:20:21.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:21 smithi189 bash[66969]: cluster 2024-03-07T03:20:20.644813+0000 mgr.y (mgr.34107) 231 : cluster 0 pgmap v440: 105 pgs: 105 active+clean; 4.6 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 263 op/s 2024-03-07T03:20:21.897 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.PoolMetadataConfApply (108 ms) 2024-03-07T03:20:21.897 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.Sparsify 2024-03-07T03:20:21.931 INFO:tasks.workunit.client.0.smithi145.stdout:sparsify_supported=1 2024-03-07T03:20:21.931 INFO:tasks.workunit.client.0.smithi145.stdout:sparse_read_supported=1 2024-03-07T03:20:21.975 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.Sparsify (78 ms) 2024-03-07T03:20:21.975 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.SparsifyClone 2024-03-07T03:20:22.000 INFO:tasks.workunit.client.0.smithi145.stdout:sparsify_supported=1 2024-03-07T03:20:22.310 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:21 smithi145 bash[72238]: [07/Mar/2024:03:20:21] ENGINE Bus STOPPING 2024-03-07T03:20:22.310 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:21 smithi145 bash[72238]: [07/Mar/2024:03:20:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:20:22.310 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:21 smithi145 bash[72238]: [07/Mar/2024:03:20:21] ENGINE Bus STOPPED 2024-03-07T03:20:22.310 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:21 smithi145 bash[72238]: [07/Mar/2024:03:20:21] ENGINE Bus STARTING 2024-03-07T03:20:22.310 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:21 smithi145 bash[72238]: [07/Mar/2024:03:20:21] ENGINE Serving on http://:::9283 2024-03-07T03:20:22.310 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:21 smithi145 bash[72238]: [07/Mar/2024:03:20:21] ENGINE Bus STARTED 2024-03-07T03:20:22.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:20:22 smithi189 bash[59814]: ts=2024-03-07T03:20:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:22 smithi189 bash[66969]: cluster 2024-03-07T03:20:21.419006+0000 mon.a (mon.0) 733 : cluster 0 osdmap e561: 8 total, 8 up, 8 in 2024-03-07T03:20:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:22 smithi189 bash[66969]: audit 2024-03-07T03:20:21.830673+0000 mon.a (mon.0) 734 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:20:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:22 smithi189 bash[66969]: audit 2024-03-07T03:20:21.851711+0000 mon.a (mon.0) 735 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:22 smithi189 bash[66969]: audit 2024-03-07T03:20:21.853893+0000 mon.a (mon.0) 736 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:22 smithi189 bash[66969]: audit 2024-03-07T03:20:21.855274+0000 mon.a (mon.0) 737 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:20:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:22 smithi189 bash[66969]: audit 2024-03-07T03:20:21.861526+0000 mon.a (mon.0) 738 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:20:22.753 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:22 smithi145 bash[79878]: cluster 2024-03-07T03:20:21.419006+0000 mon.a (mon.0) 733 : cluster 0 osdmap e561: 8 total, 8 up, 8 in 2024-03-07T03:20:22.754 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:22 smithi145 bash[79878]: audit 2024-03-07T03:20:21.830673+0000 mon.a (mon.0) 734 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:20:22.754 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:22 smithi145 bash[79878]: audit 2024-03-07T03:20:21.851711+0000 mon.a (mon.0) 735 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:22.754 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:22 smithi145 bash[79878]: audit 2024-03-07T03:20:21.853893+0000 mon.a (mon.0) 736 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:22.754 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:22 smithi145 bash[79878]: audit 2024-03-07T03:20:21.855274+0000 mon.a (mon.0) 737 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:20:22.754 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:22 smithi145 bash[79878]: audit 2024-03-07T03:20:21.861526+0000 mon.a (mon.0) 738 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:20:22.755 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:22 smithi145 bash[82329]: cluster 2024-03-07T03:20:21.419006+0000 mon.a (mon.0) 733 : cluster 0 osdmap e561: 8 total, 8 up, 8 in 2024-03-07T03:20:22.755 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:22 smithi145 bash[82329]: audit 2024-03-07T03:20:21.830673+0000 mon.a (mon.0) 734 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:20:22.755 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:22 smithi145 bash[82329]: audit 2024-03-07T03:20:21.851711+0000 mon.a (mon.0) 735 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:22.755 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:22 smithi145 bash[82329]: audit 2024-03-07T03:20:21.853893+0000 mon.a (mon.0) 736 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:22.755 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:22 smithi145 bash[82329]: audit 2024-03-07T03:20:21.855274+0000 mon.a (mon.0) 737 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:20:22.755 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:22 smithi145 bash[82329]: audit 2024-03-07T03:20:21.861526+0000 mon.a (mon.0) 738 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:20:23.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:20:22] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.43.0" 2024-03-07T03:20:23.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:23 smithi145 bash[82329]: cluster 2024-03-07T03:20:22.645765+0000 mgr.y (mgr.34107) 232 : cluster 0 pgmap v442: 105 pgs: 105 active+clean; 4.7 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.9 MiB/s wr, 215 op/s 2024-03-07T03:20:23.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:23 smithi145 bash[82329]: cluster 2024-03-07T03:20:22.868368+0000 mon.a (mon.0) 739 : cluster 0 osdmap e562: 8 total, 8 up, 8 in 2024-03-07T03:20:23.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:23 smithi145 bash[79878]: cluster 2024-03-07T03:20:22.645765+0000 mgr.y (mgr.34107) 232 : cluster 0 pgmap v442: 105 pgs: 105 active+clean; 4.7 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.9 MiB/s wr, 215 op/s 2024-03-07T03:20:23.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:23 smithi145 bash[79878]: cluster 2024-03-07T03:20:22.868368+0000 mon.a (mon.0) 739 : cluster 0 osdmap e562: 8 total, 8 up, 8 in 2024-03-07T03:20:23.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:23 smithi189 bash[66969]: cluster 2024-03-07T03:20:22.645765+0000 mgr.y (mgr.34107) 232 : cluster 0 pgmap v442: 105 pgs: 105 active+clean; 4.7 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.9 MiB/s wr, 215 op/s 2024-03-07T03:20:23.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:23 smithi189 bash[66969]: cluster 2024-03-07T03:20:22.868368+0000 mon.a (mon.0) 739 : cluster 0 osdmap e562: 8 total, 8 up, 8 in 2024-03-07T03:20:23.891 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.SparsifyClone (1915 ms) 2024-03-07T03:20:23.891 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestInternal.MissingDataPool 2024-03-07T03:20:25.095 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestInternal.MissingDataPool (1203 ms) 2024-03-07T03:20:25.243 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:24 smithi145 bash[79878]: cluster 2024-03-07T03:20:23.871458+0000 mon.a (mon.0) 740 : cluster 0 osdmap e563: 8 total, 8 up, 8 in 2024-03-07T03:20:25.244 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:24 smithi145 bash[79878]: cluster 2024-03-07T03:20:24.646521+0000 mgr.y (mgr.34107) 233 : cluster 0 pgmap v445: 105 pgs: 105 active+clean; 5.1 MiB data, 442 MiB used, 715 GiB / 715 GiB avail; 476 KiB/s rd, 171 KiB/s wr, 768 op/s 2024-03-07T03:20:25.244 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:24 smithi145 bash[82329]: cluster 2024-03-07T03:20:23.871458+0000 mon.a (mon.0) 740 : cluster 0 osdmap e563: 8 total, 8 up, 8 in 2024-03-07T03:20:25.244 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:24 smithi145 bash[82329]: cluster 2024-03-07T03:20:24.646521+0000 mgr.y (mgr.34107) 233 : cluster 0 pgmap v445: 105 pgs: 105 active+clean; 5.1 MiB data, 442 MiB used, 715 GiB / 715 GiB avail; 476 KiB/s rd, 171 KiB/s wr, 768 op/s 2024-03-07T03:20:25.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:24 smithi189 bash[66969]: cluster 2024-03-07T03:20:23.871458+0000 mon.a (mon.0) 740 : cluster 0 osdmap e563: 8 total, 8 up, 8 in 2024-03-07T03:20:25.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:24 smithi189 bash[66969]: cluster 2024-03-07T03:20:24.646521+0000 mgr.y (mgr.34107) 233 : cluster 0 pgmap v445: 105 pgs: 105 active+clean; 5.1 MiB data, 442 MiB used, 715 GiB / 715 GiB avail; 476 KiB/s rd, 171 KiB/s wr, 768 op/s 2024-03-07T03:20:25.879 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 34 tests from TestInternal (35753 ms total) 2024-03-07T03:20:25.879 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:20:25.879 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 17 tests from TestImageWatcher 2024-03-07T03:20:26.156 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:25 smithi145 bash[79878]: cluster 2024-03-07T03:20:24.882550+0000 mon.a (mon.0) 741 : cluster 0 osdmap e564: 8 total, 8 up, 8 in 2024-03-07T03:20:26.156 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:25 smithi145 bash[82329]: cluster 2024-03-07T03:20:24.882550+0000 mon.a (mon.0) 741 : cluster 0 osdmap e564: 8 total, 8 up, 8 in 2024-03-07T03:20:26.158 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade resume' 2024-03-07T03:20:26.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:25 smithi189 bash[66969]: cluster 2024-03-07T03:20:24.882550+0000 mon.a (mon.0) 741 : cluster 0 osdmap e564: 8 total, 8 up, 8 in 2024-03-07T03:20:27.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:26 smithi145 bash[79878]: cluster 2024-03-07T03:20:25.884057+0000 mon.a (mon.0) 742 : cluster 0 osdmap e565: 8 total, 8 up, 8 in 2024-03-07T03:20:27.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:26 smithi145 bash[79878]: cluster 2024-03-07T03:20:26.646997+0000 mgr.y (mgr.34107) 234 : cluster 0 pgmap v448: 73 pgs: 73 active+clean; 582 KiB data, 442 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:27.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:26 smithi145 bash[82329]: cluster 2024-03-07T03:20:25.884057+0000 mon.a (mon.0) 742 : cluster 0 osdmap e565: 8 total, 8 up, 8 in 2024-03-07T03:20:27.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:26 smithi145 bash[82329]: cluster 2024-03-07T03:20:26.646997+0000 mgr.y (mgr.34107) 234 : cluster 0 pgmap v448: 73 pgs: 73 active+clean; 582 KiB data, 442 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:27.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:26 smithi189 bash[66969]: cluster 2024-03-07T03:20:25.884057+0000 mon.a (mon.0) 742 : cluster 0 osdmap e565: 8 total, 8 up, 8 in 2024-03-07T03:20:27.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:26 smithi189 bash[66969]: cluster 2024-03-07T03:20:26.646997+0000 mgr.y (mgr.34107) 234 : cluster 0 pgmap v448: 73 pgs: 73 active+clean; 582 KiB data, 442 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:27.906 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifyHeaderUpdate 2024-03-07T03:20:28.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:27 smithi145 bash[82329]: cluster 2024-03-07T03:20:26.912553+0000 mon.a (mon.0) 743 : cluster 0 osdmap e566: 8 total, 8 up, 8 in 2024-03-07T03:20:28.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:27 smithi145 bash[82329]: audit 2024-03-07T03:20:26.915056+0000 mon.a (mon.0) 744 : audit 1 from='client.? 172.21.15.145:0/891939268' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:28.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:27 smithi145 bash[79878]: cluster 2024-03-07T03:20:26.912553+0000 mon.a (mon.0) 743 : cluster 0 osdmap e566: 8 total, 8 up, 8 in 2024-03-07T03:20:28.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:27 smithi145 bash[79878]: audit 2024-03-07T03:20:26.915056+0000 mon.a (mon.0) 744 : audit 1 from='client.? 172.21.15.145:0/891939268' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:28.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:27 smithi189 bash[66969]: cluster 2024-03-07T03:20:26.912553+0000 mon.a (mon.0) 743 : cluster 0 osdmap e566: 8 total, 8 up, 8 in 2024-03-07T03:20:28.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:27 smithi189 bash[66969]: audit 2024-03-07T03:20:26.915056+0000 mon.a (mon.0) 744 : audit 1 from='client.? 172.21.15.145:0/891939268' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:29.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:28 smithi145 bash[79878]: audit 2024-03-07T03:20:27.906952+0000 mon.a (mon.0) 745 : audit 1 from='client.? 172.21.15.145:0/891939268' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:29.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:28 smithi145 bash[79878]: cluster 2024-03-07T03:20:27.915868+0000 mon.a (mon.0) 746 : cluster 0 osdmap e567: 8 total, 8 up, 8 in 2024-03-07T03:20:29.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:28 smithi145 bash[79878]: cluster 2024-03-07T03:20:28.648060+0000 mgr.y (mgr.34107) 235 : cluster 0 pgmap v451: 105 pgs: 11 unknown, 94 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:29.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:28 smithi145 bash[82329]: audit 2024-03-07T03:20:27.906952+0000 mon.a (mon.0) 745 : audit 1 from='client.? 172.21.15.145:0/891939268' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:29.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:28 smithi145 bash[82329]: cluster 2024-03-07T03:20:27.915868+0000 mon.a (mon.0) 746 : cluster 0 osdmap e567: 8 total, 8 up, 8 in 2024-03-07T03:20:29.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:28 smithi145 bash[82329]: cluster 2024-03-07T03:20:28.648060+0000 mgr.y (mgr.34107) 235 : cluster 0 pgmap v451: 105 pgs: 11 unknown, 94 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:29.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:28 smithi189 bash[66969]: audit 2024-03-07T03:20:27.906952+0000 mon.a (mon.0) 745 : audit 1 from='client.? 172.21.15.145:0/891939268' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:29.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:28 smithi189 bash[66969]: cluster 2024-03-07T03:20:27.915868+0000 mon.a (mon.0) 746 : cluster 0 osdmap e567: 8 total, 8 up, 8 in 2024-03-07T03:20:29.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:28 smithi189 bash[66969]: cluster 2024-03-07T03:20:28.648060+0000 mgr.y (mgr.34107) 235 : cluster 0 pgmap v451: 105 pgs: 11 unknown, 94 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:29.987 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifyHeaderUpdate (2081 ms) 2024-03-07T03:20:29.987 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifyFlatten 2024-03-07T03:20:30.046 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifyFlatten (60 ms) 2024-03-07T03:20:30.046 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifyResize 2024-03-07T03:20:30.101 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifyResize (54 ms) 2024-03-07T03:20:30.101 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifyRebuildObjectMap 2024-03-07T03:20:30.149 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifyRebuildObjectMap (49 ms) 2024-03-07T03:20:30.150 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifyUpdateFeatures 2024-03-07T03:20:30.194 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifyUpdateFeatures (44 ms) 2024-03-07T03:20:30.194 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifySnapCreate 2024-03-07T03:20:30.248 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifySnapCreate (54 ms) 2024-03-07T03:20:30.248 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifySnapCreateError 2024-03-07T03:20:30.284 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifySnapCreateError (36 ms) 2024-03-07T03:20:30.284 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifySnapRename 2024-03-07T03:20:30.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:29 smithi145 bash[79878]: cluster 2024-03-07T03:20:28.933501+0000 mon.a (mon.0) 747 : cluster 0 osdmap e568: 8 total, 8 up, 8 in 2024-03-07T03:20:30.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:29 smithi145 bash[82329]: cluster 2024-03-07T03:20:28.933501+0000 mon.a (mon.0) 747 : cluster 0 osdmap e568: 8 total, 8 up, 8 in 2024-03-07T03:20:30.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:29 smithi189 bash[66969]: cluster 2024-03-07T03:20:28.933501+0000 mon.a (mon.0) 747 : cluster 0 osdmap e568: 8 total, 8 up, 8 in 2024-03-07T03:20:30.335 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifySnapRename (51 ms) 2024-03-07T03:20:30.335 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifySnapRenameError 2024-03-07T03:20:30.381 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifySnapRenameError (46 ms) 2024-03-07T03:20:30.381 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifySnapRemove 2024-03-07T03:20:30.416 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifySnapRemove (34 ms) 2024-03-07T03:20:30.416 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifySnapProtect 2024-03-07T03:20:30.454 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifySnapProtect (38 ms) 2024-03-07T03:20:30.454 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifySnapUnprotect 2024-03-07T03:20:30.503 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifySnapUnprotect (49 ms) 2024-03-07T03:20:30.503 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifyRename 2024-03-07T03:20:30.546 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifyRename (43 ms) 2024-03-07T03:20:30.546 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifyAsyncTimedOut 2024-03-07T03:20:30.589 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifyAsyncTimedOut (44 ms) 2024-03-07T03:20:30.590 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifyAsyncError 2024-03-07T03:20:30.642 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifyAsyncError (53 ms) 2024-03-07T03:20:30.643 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifyAsyncCompleteError 2024-03-07T03:20:30.684 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifyAsyncCompleteError (41 ms) 2024-03-07T03:20:30.684 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestImageWatcher.NotifyAsyncRequestTimedOut 2024-03-07T03:20:30.719 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestImageWatcher.NotifyAsyncRequestTimedOut (35 ms) 2024-03-07T03:20:30.884 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:20:30.946 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 17 tests from TestImageWatcher (2813 ms total) 2024-03-07T03:20:30.946 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:20:30.946 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 168 tests from TestLibRBD 2024-03-07T03:20:30.979 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.CreateAndStat 2024-03-07T03:20:31.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:30 smithi145 bash[79878]: cluster 2024-03-07T03:20:29.937577+0000 mon.a (mon.0) 748 : cluster 0 osdmap e569: 8 total, 8 up, 8 in 2024-03-07T03:20:31.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:30 smithi145 bash[79878]: cluster 2024-03-07T03:20:30.648655+0000 mgr.y (mgr.34107) 236 : cluster 0 pgmap v454: 105 pgs: 11 unknown, 94 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:31.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:30 smithi145 bash[82329]: cluster 2024-03-07T03:20:29.937577+0000 mon.a (mon.0) 748 : cluster 0 osdmap e569: 8 total, 8 up, 8 in 2024-03-07T03:20:31.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:30 smithi145 bash[82329]: cluster 2024-03-07T03:20:30.648655+0000 mgr.y (mgr.34107) 236 : cluster 0 pgmap v454: 105 pgs: 11 unknown, 94 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:31.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:30 smithi189 bash[66969]: cluster 2024-03-07T03:20:29.937577+0000 mon.a (mon.0) 748 : cluster 0 osdmap e569: 8 total, 8 up, 8 in 2024-03-07T03:20:31.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:30 smithi189 bash[66969]: cluster 2024-03-07T03:20:30.648655+0000 mgr.y (mgr.34107) 236 : cluster 0 pgmap v454: 105 pgs: 11 unknown, 94 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:32.211 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:31 smithi145 bash[79878]: cluster 2024-03-07T03:20:30.955877+0000 mon.a (mon.0) 749 : cluster 0 osdmap e570: 8 total, 8 up, 8 in 2024-03-07T03:20:32.211 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:31 smithi145 bash[82329]: cluster 2024-03-07T03:20:30.955877+0000 mon.a (mon.0) 749 : cluster 0 osdmap e570: 8 total, 8 up, 8 in 2024-03-07T03:20:32.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:31 smithi189 bash[66969]: cluster 2024-03-07T03:20:30.955877+0000 mon.a (mon.0) 749 : cluster 0 osdmap e570: 8 total, 8 up, 8 in 2024-03-07T03:20:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:20:32 smithi189 bash[59814]: ts=2024-03-07T03:20:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:32.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:20:32] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2024-03-07T03:20:32.986 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:33.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:32 smithi145 bash[79878]: cluster 2024-03-07T03:20:31.967500+0000 mon.a (mon.0) 750 : cluster 0 osdmap e571: 8 total, 8 up, 8 in 2024-03-07T03:20:33.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:32 smithi145 bash[79878]: audit 2024-03-07T03:20:31.970251+0000 mon.c (mon.2) 9 : audit 1 from='client.? 172.21.15.145:0/1005014357' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:33.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:32 smithi145 bash[79878]: audit 2024-03-07T03:20:31.972404+0000 mon.a (mon.0) 751 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:33.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:32 smithi145 bash[79878]: audit 2024-03-07T03:20:32.639797+0000 mon.a (mon.0) 752 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:33.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:32 smithi145 bash[79878]: cluster 2024-03-07T03:20:32.651379+0000 mgr.y (mgr.34107) 237 : cluster 0 pgmap v457: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 431 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:33.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:32 smithi145 bash[82329]: cluster 2024-03-07T03:20:31.967500+0000 mon.a (mon.0) 750 : cluster 0 osdmap e571: 8 total, 8 up, 8 in 2024-03-07T03:20:33.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:32 smithi145 bash[82329]: audit 2024-03-07T03:20:31.970251+0000 mon.c (mon.2) 9 : audit 1 from='client.? 172.21.15.145:0/1005014357' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:33.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:32 smithi145 bash[82329]: audit 2024-03-07T03:20:31.972404+0000 mon.a (mon.0) 751 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:33.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:32 smithi145 bash[82329]: audit 2024-03-07T03:20:32.639797+0000 mon.a (mon.0) 752 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:33.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:32 smithi145 bash[82329]: cluster 2024-03-07T03:20:32.651379+0000 mgr.y (mgr.34107) 237 : cluster 0 pgmap v457: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 431 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:33.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:32 smithi189 bash[66969]: cluster 2024-03-07T03:20:31.967500+0000 mon.a (mon.0) 750 : cluster 0 osdmap e571: 8 total, 8 up, 8 in 2024-03-07T03:20:33.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:32 smithi189 bash[66969]: audit 2024-03-07T03:20:31.970251+0000 mon.c (mon.2) 9 : audit 1 from='client.? 172.21.15.145:0/1005014357' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:33.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:32 smithi189 bash[66969]: audit 2024-03-07T03:20:31.972404+0000 mon.a (mon.0) 751 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:33.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:32 smithi189 bash[66969]: audit 2024-03-07T03:20:32.639797+0000 mon.a (mon.0) 752 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:33.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:32 smithi189 bash[66969]: cluster 2024-03-07T03:20:32.651379+0000 mgr.y (mgr.34107) 237 : cluster 0 pgmap v457: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 431 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:34.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:33 smithi145 bash[79878]: cluster 2024-03-07T03:20:32.961715+0000 mon.a (mon.0) 753 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:20:34.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:33 smithi145 bash[79878]: audit 2024-03-07T03:20:32.974548+0000 mon.a (mon.0) 754 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:34.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:33 smithi145 bash[79878]: cluster 2024-03-07T03:20:32.985238+0000 mon.a (mon.0) 755 : cluster 0 osdmap e572: 8 total, 8 up, 8 in 2024-03-07T03:20:34.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:33 smithi145 bash[82329]: cluster 2024-03-07T03:20:32.961715+0000 mon.a (mon.0) 753 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:20:34.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:33 smithi145 bash[82329]: audit 2024-03-07T03:20:32.974548+0000 mon.a (mon.0) 754 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:34.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:33 smithi145 bash[82329]: cluster 2024-03-07T03:20:32.985238+0000 mon.a (mon.0) 755 : cluster 0 osdmap e572: 8 total, 8 up, 8 in 2024-03-07T03:20:34.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:33 smithi189 bash[66969]: cluster 2024-03-07T03:20:32.961715+0000 mon.a (mon.0) 753 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:20:34.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:33 smithi189 bash[66969]: audit 2024-03-07T03:20:32.974548+0000 mon.a (mon.0) 754 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:34.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:33 smithi189 bash[66969]: cluster 2024-03-07T03:20:32.985238+0000 mon.a (mon.0) 755 : cluster 0 osdmap e572: 8 total, 8 up, 8 in 2024-03-07T03:20:34.579 INFO:teuthology.orchestra.run.smithi145.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:20:34.983 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[72238]: debug 2024-03-07T03:20:34.692+0000 7f318c55a700 -1 mgr.server reply reply (11) Resource temporarily unavailable 6 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:20:34.983 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[72238]: debug 2024-03-07T03:20:34.692+0000 7f318c55a700 -1 mgr.server reply reply (16) Device or resource busy 6 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-03-07T03:20:35.059 INFO:tasks.workunit.client.0.smithi145.stdout:image has size 2097152 and order 22 2024-03-07T03:20:35.060 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.CreateAndStat (4081 ms) 2024-03-07T03:20:35.060 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.CreateWithSameDataPool 2024-03-07T03:20:35.060 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.096 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.CreateWithSameDataPool (36 ms) 2024-03-07T03:20:35.096 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.CreateAndStatPP 2024-03-07T03:20:35.097 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.139 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.CreateAndStatPP (43 ms) 2024-03-07T03:20:35.139 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.GetId 2024-03-07T03:20:35.139 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.185 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.GetId (46 ms) 2024-03-07T03:20:35.186 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.GetIdPP 2024-03-07T03:20:35.186 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.241 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.GetIdPP (55 ms) 2024-03-07T03:20:35.241 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefix 2024-03-07T03:20:35.241 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.276 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.GetBlockNamePrefix (36 ms) 2024-03-07T03:20:35.277 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefixPP 2024-03-07T03:20:35.277 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: cluster 2024-03-07T03:20:33.986008+0000 mon.a (mon.0) 756 : cluster 0 osdmap e573: 8 total, 8 up, 8 in 2024-03-07T03:20:35.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: audit 2024-03-07T03:20:34.582746+0000 mgr.y (mgr.34107) 238 : audit 0 from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:20:35.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: cephadm 2024-03-07T03:20:34.583411+0000 mgr.y (mgr.34107) 239 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:20:35.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: audit 2024-03-07T03:20:34.590251+0000 mon.a (mon.0) 757 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:20:35.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: audit 2024-03-07T03:20:34.620622+0000 mon.a (mon.0) 758 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:35.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: audit 2024-03-07T03:20:34.622447+0000 mon.a (mon.0) 759 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:35.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: audit 2024-03-07T03:20:34.623163+0000 mon.a (mon.0) 760 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:20:35.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: audit 2024-03-07T03:20:34.630348+0000 mon.a (mon.0) 761 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:20:35.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: cluster 2024-03-07T03:20:34.651847+0000 mgr.y (mgr.34107) 240 : cluster 0 pgmap v460: 105 pgs: 6 creating+activating, 6 unknown, 93 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:35.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: audit 2024-03-07T03:20:34.689515+0000 mon.a (mon.0) 762 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:35.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: audit 2024-03-07T03:20:34.691488+0000 mon.a (mon.0) 763 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:35.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: audit 2024-03-07T03:20:34.693098+0000 mon.a (mon.0) 764 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:35.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: audit 2024-03-07T03:20:34.694169+0000 mon.a (mon.0) 765 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:35.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: audit 2024-03-07T03:20:34.695654+0000 mon.a (mon.0) 766 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:20:35.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: audit 2024-03-07T03:20:34.695801+0000 mgr.y (mgr.34107) 241 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:20:35.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[79878]: cephadm 2024-03-07T03:20:34.696543+0000 mgr.y (mgr.34107) 242 : cephadm 1 Upgrade: 6 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:20:35.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[82329]: cluster 2024-03-07T03:20:33.986008+0000 mon.a (mon.0) 756 : cluster 0 osdmap e573: 8 total, 8 up, 8 in 2024-03-07T03:20:35.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[82329]: audit 2024-03-07T03:20:34.582746+0000 mgr.y (mgr.34107) 238 : audit 0 from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:20:35.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[82329]: cephadm 2024-03-07T03:20:34.583411+0000 mgr.y (mgr.34107) 239 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:20:35.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[82329]: audit 2024-03-07T03:20:34.590251+0000 mon.a (mon.0) 757 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:20:35.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[82329]: audit 2024-03-07T03:20:34.620622+0000 mon.a (mon.0) 758 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:35.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[82329]: audit 2024-03-07T03:20:34.622447+0000 mon.a (mon.0) 759 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:35.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[82329]: audit 2024-03-07T03:20:34.623163+0000 mon.a (mon.0) 760 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:20:35.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[82329]: audit 2024-03-07T03:20:34.630348+0000 mon.a (mon.0) 761 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:20:35.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[82329]: cluster 2024-03-07T03:20:34.651847+0000 mgr.y (mgr.34107) 240 : cluster 0 pgmap v460: 105 pgs: 6 creating+activating, 6 unknown, 93 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:35.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[82329]: audit 2024-03-07T03:20:34.689515+0000 mon.a (mon.0) 762 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:35.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[82329]: audit 2024-03-07T03:20:34.691488+0000 mon.a (mon.0) 763 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:35.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:34 smithi145 bash[82329]: audit 2024-03-07T03:20:34.693098+0000 mon.a (mon.0) 764 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:35.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:35 smithi145 bash[82329]: audit 2024-03-07T03:20:34.694169+0000 mon.a (mon.0) 765 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:35.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:35 smithi145 bash[82329]: audit 2024-03-07T03:20:34.695654+0000 mon.a (mon.0) 766 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:20:35.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:35 smithi145 bash[82329]: audit 2024-03-07T03:20:34.695801+0000 mgr.y (mgr.34107) 241 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:20:35.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:35 smithi145 bash[82329]: cephadm 2024-03-07T03:20:34.696543+0000 mgr.y (mgr.34107) 242 : cephadm 1 Upgrade: 6 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:20:35.313 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.GetBlockNamePrefixPP (36 ms) 2024-03-07T03:20:35.314 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestGetCreateTimestamp 2024-03-07T03:20:35.314 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: cluster 2024-03-07T03:20:33.986008+0000 mon.a (mon.0) 756 : cluster 0 osdmap e573: 8 total, 8 up, 8 in 2024-03-07T03:20:35.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: audit 2024-03-07T03:20:34.582746+0000 mgr.y (mgr.34107) 238 : audit 0 from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:20:35.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: cephadm 2024-03-07T03:20:34.583411+0000 mgr.y (mgr.34107) 239 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:20:35.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: audit 2024-03-07T03:20:34.590251+0000 mon.a (mon.0) 757 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:20:35.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: audit 2024-03-07T03:20:34.620622+0000 mon.a (mon.0) 758 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:35.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: audit 2024-03-07T03:20:34.622447+0000 mon.a (mon.0) 759 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:35.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: audit 2024-03-07T03:20:34.623163+0000 mon.a (mon.0) 760 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:20:35.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: audit 2024-03-07T03:20:34.630348+0000 mon.a (mon.0) 761 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:20:35.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: cluster 2024-03-07T03:20:34.651847+0000 mgr.y (mgr.34107) 240 : cluster 0 pgmap v460: 105 pgs: 6 creating+activating, 6 unknown, 93 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:35.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: audit 2024-03-07T03:20:34.689515+0000 mon.a (mon.0) 762 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:35.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: audit 2024-03-07T03:20:34.691488+0000 mon.a (mon.0) 763 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:35.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: audit 2024-03-07T03:20:34.693098+0000 mon.a (mon.0) 764 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:35.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: audit 2024-03-07T03:20:34.694169+0000 mon.a (mon.0) 765 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:35.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: audit 2024-03-07T03:20:34.695654+0000 mon.a (mon.0) 766 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:20:35.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: audit 2024-03-07T03:20:34.695801+0000 mgr.y (mgr.34107) 241 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:20:35.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:34 smithi189 bash[66969]: cephadm 2024-03-07T03:20:34.696543+0000 mgr.y (mgr.34107) 242 : cephadm 1 Upgrade: 6 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:20:35.350 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestGetCreateTimestamp (37 ms) 2024-03-07T03:20:35.350 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.GetCreateTimestampPP 2024-03-07T03:20:35.350 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.389 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.GetCreateTimestampPP (39 ms) 2024-03-07T03:20:35.389 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.OpenAio 2024-03-07T03:20:35.389 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.433 INFO:tasks.workunit.client.0.smithi145.stdout:image has size 2097152 and order 22 2024-03-07T03:20:35.433 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.OpenAio (44 ms) 2024-03-07T03:20:35.433 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.OpenAioFail 2024-03-07T03:20:35.441 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.OpenAioFail (8 ms) 2024-03-07T03:20:35.441 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.OpenAioPP 2024-03-07T03:20:35.441 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.449 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'echo wait for half of osds to upgrade' 2024-03-07T03:20:35.503 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.OpenAioPP (62 ms) 2024-03-07T03:20:35.503 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.OpenAioFailPP 2024-03-07T03:20:35.509 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.OpenAioFailPP (6 ms) 2024-03-07T03:20:35.509 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ResizeAndStat 2024-03-07T03:20:35.509 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.592 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ResizeAndStat (83 ms) 2024-03-07T03:20:35.592 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ResizeAndStatPP 2024-03-07T03:20:35.592 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.650 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ResizeAndStatPP (58 ms) 2024-03-07T03:20:35.650 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResize 2024-03-07T03:20:35.650 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.727 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.UpdateWatchAndResize (77 ms) 2024-03-07T03:20:35.728 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResizePP 2024-03-07T03:20:35.728 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:35.809 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.UpdateWatchAndResizePP (82 ms) 2024-03-07T03:20:35.809 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCreateLsDelete 2024-03-07T03:20:36.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:36 smithi145 bash[79878]: cluster 2024-03-07T03:20:35.012962+0000 mon.a (mon.0) 767 : cluster 0 osdmap e574: 8 total, 8 up, 8 in 2024-03-07T03:20:36.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:36 smithi145 bash[79878]: cluster 2024-03-07T03:20:35.628315+0000 mon.a (mon.0) 768 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:20:36.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:36 smithi145 bash[82329]: cluster 2024-03-07T03:20:35.012962+0000 mon.a (mon.0) 767 : cluster 0 osdmap e574: 8 total, 8 up, 8 in 2024-03-07T03:20:36.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:36 smithi145 bash[82329]: cluster 2024-03-07T03:20:35.628315+0000 mon.a (mon.0) 768 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:20:36.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:36 smithi189 bash[66969]: cluster 2024-03-07T03:20:35.012962+0000 mon.a (mon.0) 767 : cluster 0 osdmap e574: 8 total, 8 up, 8 in 2024-03-07T03:20:36.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:36 smithi189 bash[66969]: cluster 2024-03-07T03:20:35.628315+0000 mon.a (mon.0) 768 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:20:37.676 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:38.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:37 smithi145 bash[79878]: cluster 2024-03-07T03:20:36.640953+0000 mon.a (mon.0) 769 : cluster 0 osdmap e575: 8 total, 8 up, 8 in 2024-03-07T03:20:38.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:37 smithi145 bash[79878]: audit 2024-03-07T03:20:36.643115+0000 mon.a (mon.0) 770 : audit 1 from='client.? 172.21.15.145:0/3695257247' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:38.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:37 smithi145 bash[79878]: cluster 2024-03-07T03:20:36.652796+0000 mgr.y (mgr.34107) 243 : cluster 0 pgmap v463: 137 pgs: 32 unknown, 6 creating+activating, 99 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 3.5 KiB/s wr, 29 op/s 2024-03-07T03:20:38.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:37 smithi145 bash[82329]: cluster 2024-03-07T03:20:36.640953+0000 mon.a (mon.0) 769 : cluster 0 osdmap e575: 8 total, 8 up, 8 in 2024-03-07T03:20:38.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:37 smithi145 bash[82329]: audit 2024-03-07T03:20:36.643115+0000 mon.a (mon.0) 770 : audit 1 from='client.? 172.21.15.145:0/3695257247' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:38.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:37 smithi145 bash[82329]: cluster 2024-03-07T03:20:36.652796+0000 mgr.y (mgr.34107) 243 : cluster 0 pgmap v463: 137 pgs: 32 unknown, 6 creating+activating, 99 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 3.5 KiB/s wr, 29 op/s 2024-03-07T03:20:38.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:37 smithi189 bash[66969]: cluster 2024-03-07T03:20:36.640953+0000 mon.a (mon.0) 769 : cluster 0 osdmap e575: 8 total, 8 up, 8 in 2024-03-07T03:20:38.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:37 smithi189 bash[66969]: audit 2024-03-07T03:20:36.643115+0000 mon.a (mon.0) 770 : audit 1 from='client.? 172.21.15.145:0/3695257247' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:38.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:37 smithi189 bash[66969]: cluster 2024-03-07T03:20:36.652796+0000 mgr.y (mgr.34107) 243 : cluster 0 pgmap v463: 137 pgs: 32 unknown, 6 creating+activating, 99 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 3.5 KiB/s wr, 29 op/s 2024-03-07T03:20:39.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:38 smithi145 bash[79878]: cluster 2024-03-07T03:20:37.634353+0000 mon.a (mon.0) 771 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:20:39.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:38 smithi145 bash[79878]: audit 2024-03-07T03:20:37.644042+0000 mon.a (mon.0) 772 : audit 1 from='client.? 172.21.15.145:0/3695257247' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:39.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:38 smithi145 bash[79878]: cluster 2024-03-07T03:20:37.651491+0000 mon.a (mon.0) 773 : cluster 0 osdmap e576: 8 total, 8 up, 8 in 2024-03-07T03:20:39.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:38 smithi145 bash[82329]: cluster 2024-03-07T03:20:37.634353+0000 mon.a (mon.0) 771 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:20:39.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:38 smithi145 bash[82329]: audit 2024-03-07T03:20:37.644042+0000 mon.a (mon.0) 772 : audit 1 from='client.? 172.21.15.145:0/3695257247' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:39.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:38 smithi145 bash[82329]: cluster 2024-03-07T03:20:37.651491+0000 mon.a (mon.0) 773 : cluster 0 osdmap e576: 8 total, 8 up, 8 in 2024-03-07T03:20:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:38 smithi189 bash[66969]: cluster 2024-03-07T03:20:37.634353+0000 mon.a (mon.0) 771 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:20:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:38 smithi189 bash[66969]: audit 2024-03-07T03:20:37.644042+0000 mon.a (mon.0) 772 : audit 1 from='client.? 172.21.15.145:0/3695257247' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:38 smithi189 bash[66969]: cluster 2024-03-07T03:20:37.651491+0000 mon.a (mon.0) 773 : cluster 0 osdmap e576: 8 total, 8 up, 8 in 2024-03-07T03:20:39.699 INFO:tasks.workunit.client.0.smithi145.stdout:image: image18 2024-03-07T03:20:39.699 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image18 2024-03-07T03:20:39.699 INFO:tasks.workunit.client.0.smithi145.stdout:found image18 2024-03-07T03:20:39.699 INFO:tasks.workunit.client.0.smithi145.stdout:erased image18 2024-03-07T03:20:39.699 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:39.879 INFO:tasks.workunit.client.0.smithi145.stdout:image: image18 2024-03-07T03:20:39.879 INFO:tasks.workunit.client.0.smithi145.stdout:image: image19 2024-03-07T03:20:39.879 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image18 2024-03-07T03:20:39.879 INFO:tasks.workunit.client.0.smithi145.stdout:found image18 2024-03-07T03:20:39.879 INFO:tasks.workunit.client.0.smithi145.stdout:erased image18 2024-03-07T03:20:39.879 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image19 2024-03-07T03:20:39.879 INFO:tasks.workunit.client.0.smithi145.stdout:found image19 2024-03-07T03:20:39.880 INFO:tasks.workunit.client.0.smithi145.stdout:erased image19 2024-03-07T03:20:39.880 INFO:tasks.workunit.client.0.smithi145.stdout:image: image19 2024-03-07T03:20:39.880 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image19 2024-03-07T03:20:39.880 INFO:tasks.workunit.client.0.smithi145.stdout:found image19 2024-03-07T03:20:39.880 INFO:tasks.workunit.client.0.smithi145.stdout:erased image19 2024-03-07T03:20:39.880 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCreateLsDelete (4070 ms) 2024-03-07T03:20:39.880 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCreateLsDeletePP 2024-03-07T03:20:40.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:39 smithi145 bash[79878]: cluster 2024-03-07T03:20:38.654349+0000 mgr.y (mgr.34107) 244 : cluster 0 pgmap v465: 137 pgs: 3 creating+peering, 4 unknown, 130 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 155 KiB/s rd, 17 KiB/s wr, 214 op/s 2024-03-07T03:20:40.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:39 smithi145 bash[79878]: cluster 2024-03-07T03:20:38.663874+0000 mon.a (mon.0) 774 : cluster 0 osdmap e577: 8 total, 8 up, 8 in 2024-03-07T03:20:40.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:39 smithi145 bash[79878]: cluster 2024-03-07T03:20:39.645937+0000 mon.a (mon.0) 775 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:20:40.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:39 smithi145 bash[82329]: cluster 2024-03-07T03:20:38.654349+0000 mgr.y (mgr.34107) 244 : cluster 0 pgmap v465: 137 pgs: 3 creating+peering, 4 unknown, 130 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 155 KiB/s rd, 17 KiB/s wr, 214 op/s 2024-03-07T03:20:40.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:39 smithi145 bash[82329]: cluster 2024-03-07T03:20:38.663874+0000 mon.a (mon.0) 774 : cluster 0 osdmap e577: 8 total, 8 up, 8 in 2024-03-07T03:20:40.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:39 smithi145 bash[82329]: cluster 2024-03-07T03:20:39.645937+0000 mon.a (mon.0) 775 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:20:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:39 smithi189 bash[66969]: cluster 2024-03-07T03:20:38.654349+0000 mgr.y (mgr.34107) 244 : cluster 0 pgmap v465: 137 pgs: 3 creating+peering, 4 unknown, 130 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 155 KiB/s rd, 17 KiB/s wr, 214 op/s 2024-03-07T03:20:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:39 smithi189 bash[66969]: cluster 2024-03-07T03:20:38.663874+0000 mon.a (mon.0) 774 : cluster 0 osdmap e577: 8 total, 8 up, 8 in 2024-03-07T03:20:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:39 smithi189 bash[66969]: cluster 2024-03-07T03:20:39.645937+0000 mon.a (mon.0) 775 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:20:40.216 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:20:41.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:40 smithi145 bash[79878]: cluster 2024-03-07T03:20:39.674164+0000 mon.a (mon.0) 776 : cluster 0 osdmap e578: 8 total, 8 up, 8 in 2024-03-07T03:20:41.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:40 smithi145 bash[82329]: cluster 2024-03-07T03:20:39.674164+0000 mon.a (mon.0) 776 : cluster 0 osdmap e578: 8 total, 8 up, 8 in 2024-03-07T03:20:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:40 smithi189 bash[66969]: cluster 2024-03-07T03:20:39.674164+0000 mon.a (mon.0) 776 : cluster 0 osdmap e578: 8 total, 8 up, 8 in 2024-03-07T03:20:41.689 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:41.932 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:41 smithi145 bash[79878]: cluster 2024-03-07T03:20:40.655094+0000 mgr.y (mgr.34107) 245 : cluster 0 pgmap v468: 137 pgs: 3 creating+peering, 4 unknown, 130 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 165 KiB/s rd, 16 KiB/s wr, 220 op/s 2024-03-07T03:20:41.932 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:41 smithi145 bash[79878]: cluster 2024-03-07T03:20:40.686377+0000 mon.a (mon.0) 777 : cluster 0 osdmap e579: 8 total, 8 up, 8 in 2024-03-07T03:20:41.932 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:41 smithi145 bash[79878]: audit 2024-03-07T03:20:40.691091+0000 mon.a (mon.0) 778 : audit 1 from='client.? 172.21.15.145:0/1196766860' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:41.932 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:41 smithi145 bash[82329]: cluster 2024-03-07T03:20:40.655094+0000 mgr.y (mgr.34107) 245 : cluster 0 pgmap v468: 137 pgs: 3 creating+peering, 4 unknown, 130 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 165 KiB/s rd, 16 KiB/s wr, 220 op/s 2024-03-07T03:20:41.932 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:41 smithi145 bash[82329]: cluster 2024-03-07T03:20:40.686377+0000 mon.a (mon.0) 777 : cluster 0 osdmap e579: 8 total, 8 up, 8 in 2024-03-07T03:20:41.932 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:41 smithi145 bash[82329]: audit 2024-03-07T03:20:40.691091+0000 mon.a (mon.0) 778 : audit 1 from='client.? 172.21.15.145:0/1196766860' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:42.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:41 smithi189 bash[66969]: cluster 2024-03-07T03:20:40.655094+0000 mgr.y (mgr.34107) 245 : cluster 0 pgmap v468: 137 pgs: 3 creating+peering, 4 unknown, 130 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 165 KiB/s rd, 16 KiB/s wr, 220 op/s 2024-03-07T03:20:42.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:41 smithi189 bash[66969]: cluster 2024-03-07T03:20:40.686377+0000 mon.a (mon.0) 777 : cluster 0 osdmap e579: 8 total, 8 up, 8 in 2024-03-07T03:20:42.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:41 smithi189 bash[66969]: audit 2024-03-07T03:20:40.691091+0000 mon.a (mon.0) 778 : audit 1 from='client.? 172.21.15.145:0/1196766860' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:20:42 smithi189 bash[59814]: ts=2024-03-07T03:20:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:42.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:20:42] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2024-03-07T03:20:42.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:42 smithi145 bash[79878]: audit 2024-03-07T03:20:41.679724+0000 mon.a (mon.0) 779 : audit 1 from='client.? 172.21.15.145:0/1196766860' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:42.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:42 smithi145 bash[79878]: cluster 2024-03-07T03:20:41.683092+0000 mon.a (mon.0) 780 : cluster 0 osdmap e580: 8 total, 8 up, 8 in 2024-03-07T03:20:42.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:42 smithi145 bash[79878]: cluster 2024-03-07T03:20:42.345079+0000 mon.a (mon.0) 781 : cluster 0 osdmap e581: 8 total, 8 up, 8 in 2024-03-07T03:20:42.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:42 smithi145 bash[82329]: audit 2024-03-07T03:20:41.679724+0000 mon.a (mon.0) 779 : audit 1 from='client.? 172.21.15.145:0/1196766860' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:42.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:42 smithi145 bash[82329]: cluster 2024-03-07T03:20:41.683092+0000 mon.a (mon.0) 780 : cluster 0 osdmap e580: 8 total, 8 up, 8 in 2024-03-07T03:20:42.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:42 smithi145 bash[82329]: cluster 2024-03-07T03:20:42.345079+0000 mon.a (mon.0) 781 : cluster 0 osdmap e581: 8 total, 8 up, 8 in 2024-03-07T03:20:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:42 smithi189 bash[66969]: audit 2024-03-07T03:20:41.679724+0000 mon.a (mon.0) 779 : audit 1 from='client.? 172.21.15.145:0/1196766860' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:42 smithi189 bash[66969]: cluster 2024-03-07T03:20:41.683092+0000 mon.a (mon.0) 780 : cluster 0 osdmap e580: 8 total, 8 up, 8 in 2024-03-07T03:20:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:42 smithi189 bash[66969]: cluster 2024-03-07T03:20:42.345079+0000 mon.a (mon.0) 781 : cluster 0 osdmap e581: 8 total, 8 up, 8 in 2024-03-07T03:20:43.384 INFO:tasks.workunit.client.0.smithi145.stdout:num images is: 1 2024-03-07T03:20:43.384 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:20:43.384 INFO:tasks.workunit.client.0.smithi145.stdout:image: image20 2024-03-07T03:20:43.384 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image20 2024-03-07T03:20:43.385 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:43.408 INFO:tasks.workunit.client.0.smithi145.stdout:num images is: 2 2024-03-07T03:20:43.408 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 2 2024-03-07T03:20:43.408 INFO:tasks.workunit.client.0.smithi145.stdout:image: image20 2024-03-07T03:20:43.408 INFO:tasks.workunit.client.0.smithi145.stdout:image: image21 2024-03-07T03:20:43.408 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image20 2024-03-07T03:20:43.408 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image21 2024-03-07T03:20:43.510 INFO:tasks.workunit.client.0.smithi145.stdout:num images is: 1 2024-03-07T03:20:43.510 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:20:43.510 INFO:tasks.workunit.client.0.smithi145.stdout:image: image21 2024-03-07T03:20:43.510 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image21 2024-03-07T03:20:43.510 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCreateLsDeletePP (3631 ms) 2024-03-07T03:20:43.510 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCopy 2024-03-07T03:20:43.518 INFO:teuthology.orchestra.run.smithi145.stdout:wait for half of osds to upgrade 2024-03-07T03:20:44.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:43 smithi145 bash[79878]: cluster 2024-03-07T03:20:42.655816+0000 mgr.y (mgr.34107) 246 : cluster 0 pgmap v472: 169 pgs: 3 creating+peering, 26 unknown, 140 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 KiB/s wr, 3 op/s 2024-03-07T03:20:44.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:43 smithi145 bash[79878]: cluster 2024-03-07T03:20:43.353176+0000 mon.a (mon.0) 782 : cluster 0 osdmap e582: 8 total, 8 up, 8 in 2024-03-07T03:20:44.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:43 smithi145 bash[82329]: cluster 2024-03-07T03:20:42.655816+0000 mgr.y (mgr.34107) 246 : cluster 0 pgmap v472: 169 pgs: 3 creating+peering, 26 unknown, 140 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 KiB/s wr, 3 op/s 2024-03-07T03:20:44.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:43 smithi145 bash[82329]: cluster 2024-03-07T03:20:43.353176+0000 mon.a (mon.0) 782 : cluster 0 osdmap e582: 8 total, 8 up, 8 in 2024-03-07T03:20:44.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:43 smithi189 bash[66969]: cluster 2024-03-07T03:20:42.655816+0000 mgr.y (mgr.34107) 246 : cluster 0 pgmap v472: 169 pgs: 3 creating+peering, 26 unknown, 140 active+clean; 583 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1 KiB/s wr, 3 op/s 2024-03-07T03:20:44.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:43 smithi189 bash[66969]: cluster 2024-03-07T03:20:43.353176+0000 mon.a (mon.0) 782 : cluster 0 osdmap e582: 8 total, 8 up, 8 in 2024-03-07T03:20:44.452 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-03-07T03:20:45.365 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:45.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:45 smithi145 bash[79878]: cluster 2024-03-07T03:20:44.365968+0000 mon.a (mon.0) 783 : cluster 0 osdmap e583: 8 total, 8 up, 8 in 2024-03-07T03:20:45.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:45 smithi145 bash[79878]: audit 2024-03-07T03:20:44.372299+0000 mon.c (mon.2) 10 : audit 1 from='client.? 172.21.15.145:0/3118372268' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:45.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:45 smithi145 bash[79878]: audit 2024-03-07T03:20:44.374543+0000 mon.a (mon.0) 784 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:45.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:45 smithi145 bash[79878]: cluster 2024-03-07T03:20:44.656451+0000 mgr.y (mgr.34107) 247 : cluster 0 pgmap v475: 201 pgs: 32 unknown, 3 creating+peering, 166 active+clean; 583 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 5.5 KiB/s wr, 76 op/s 2024-03-07T03:20:45.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:45 smithi145 bash[82329]: cluster 2024-03-07T03:20:44.365968+0000 mon.a (mon.0) 783 : cluster 0 osdmap e583: 8 total, 8 up, 8 in 2024-03-07T03:20:45.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:45 smithi145 bash[82329]: audit 2024-03-07T03:20:44.372299+0000 mon.c (mon.2) 10 : audit 1 from='client.? 172.21.15.145:0/3118372268' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:45.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:45 smithi145 bash[82329]: audit 2024-03-07T03:20:44.374543+0000 mon.a (mon.0) 784 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:45.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:45 smithi145 bash[82329]: cluster 2024-03-07T03:20:44.656451+0000 mgr.y (mgr.34107) 247 : cluster 0 pgmap v475: 201 pgs: 32 unknown, 3 creating+peering, 166 active+clean; 583 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 5.5 KiB/s wr, 76 op/s 2024-03-07T03:20:45.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:45 smithi189 bash[66969]: cluster 2024-03-07T03:20:44.365968+0000 mon.a (mon.0) 783 : cluster 0 osdmap e583: 8 total, 8 up, 8 in 2024-03-07T03:20:45.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:45 smithi189 bash[66969]: audit 2024-03-07T03:20:44.372299+0000 mon.c (mon.2) 10 : audit 1 from='client.? 172.21.15.145:0/3118372268' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:45.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:45 smithi189 bash[66969]: audit 2024-03-07T03:20:44.374543+0000 mon.a (mon.0) 784 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:45.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:45 smithi189 bash[66969]: cluster 2024-03-07T03:20:44.656451+0000 mgr.y (mgr.34107) 247 : cluster 0 pgmap v475: 201 pgs: 32 unknown, 3 creating+peering, 166 active+clean; 583 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 5.5 KiB/s wr, 76 op/s 2024-03-07T03:20:46.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:46 smithi145 bash[79878]: cluster 2024-03-07T03:20:45.351796+0000 mon.a (mon.0) 785 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:20:46.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:46 smithi145 bash[79878]: audit 2024-03-07T03:20:45.361692+0000 mon.a (mon.0) 786 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:46.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:46 smithi145 bash[79878]: cluster 2024-03-07T03:20:45.371399+0000 mon.a (mon.0) 787 : cluster 0 osdmap e584: 8 total, 8 up, 8 in 2024-03-07T03:20:46.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:46 smithi145 bash[82329]: cluster 2024-03-07T03:20:45.351796+0000 mon.a (mon.0) 785 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:20:46.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:46 smithi145 bash[82329]: audit 2024-03-07T03:20:45.361692+0000 mon.a (mon.0) 786 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:46.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:46 smithi145 bash[82329]: cluster 2024-03-07T03:20:45.371399+0000 mon.a (mon.0) 787 : cluster 0 osdmap e584: 8 total, 8 up, 8 in 2024-03-07T03:20:46.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:46 smithi189 bash[66969]: cluster 2024-03-07T03:20:45.351796+0000 mon.a (mon.0) 785 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:20:46.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:46 smithi189 bash[66969]: audit 2024-03-07T03:20:45.361692+0000 mon.a (mon.0) 786 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:46.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:46 smithi189 bash[66969]: cluster 2024-03-07T03:20:45.371399+0000 mon.a (mon.0) 787 : cluster 0 osdmap e584: 8 total, 8 up, 8 in 2024-03-07T03:20:47.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:47 smithi145 bash[82329]: cluster 2024-03-07T03:20:46.376331+0000 mon.a (mon.0) 788 : cluster 0 osdmap e585: 8 total, 8 up, 8 in 2024-03-07T03:20:47.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:47 smithi145 bash[82329]: cluster 2024-03-07T03:20:46.657418+0000 mgr.y (mgr.34107) 248 : cluster 0 pgmap v478: 201 pgs: 24 unknown, 177 active+clean; 583 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 5.7 KiB/s wr, 78 op/s 2024-03-07T03:20:47.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:47 smithi145 bash[82329]: cluster 2024-03-07T03:20:47.344659+0000 mon.a (mon.0) 789 : cluster 0 osdmap e586: 8 total, 8 up, 8 in 2024-03-07T03:20:47.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:47 smithi145 bash[79878]: cluster 2024-03-07T03:20:46.376331+0000 mon.a (mon.0) 788 : cluster 0 osdmap e585: 8 total, 8 up, 8 in 2024-03-07T03:20:47.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:47 smithi145 bash[79878]: cluster 2024-03-07T03:20:46.657418+0000 mgr.y (mgr.34107) 248 : cluster 0 pgmap v478: 201 pgs: 24 unknown, 177 active+clean; 583 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 5.7 KiB/s wr, 78 op/s 2024-03-07T03:20:47.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:47 smithi145 bash[79878]: cluster 2024-03-07T03:20:47.344659+0000 mon.a (mon.0) 789 : cluster 0 osdmap e586: 8 total, 8 up, 8 in 2024-03-07T03:20:47.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:47 smithi189 bash[66969]: cluster 2024-03-07T03:20:46.376331+0000 mon.a (mon.0) 788 : cluster 0 osdmap e585: 8 total, 8 up, 8 in 2024-03-07T03:20:47.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:47 smithi189 bash[66969]: cluster 2024-03-07T03:20:46.657418+0000 mgr.y (mgr.34107) 248 : cluster 0 pgmap v478: 201 pgs: 24 unknown, 177 active+clean; 583 KiB data, 446 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 5.7 KiB/s wr, 78 op/s 2024-03-07T03:20:47.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:47 smithi189 bash[66969]: cluster 2024-03-07T03:20:47.344659+0000 mon.a (mon.0) 789 : cluster 0 osdmap e586: 8 total, 8 up, 8 in 2024-03-07T03:20:48.371 INFO:tasks.workunit.client.0.smithi145.stdout:image: image22 2024-03-07T03:20:48.371 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image22 2024-03-07T03:20:48.371 INFO:tasks.workunit.client.0.smithi145.stdout:found image22 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:erased image22 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:image: image22 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:image: image23 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image22 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:found image22 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:erased image22 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image23 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:found image23 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:erased image23 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:100.00% done 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:image: image22 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:image: image23 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:image: image24 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image22 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:found image22 2024-03-07T03:20:48.372 INFO:tasks.workunit.client.0.smithi145.stdout:erased image22 2024-03-07T03:20:48.373 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image23 2024-03-07T03:20:48.373 INFO:tasks.workunit.client.0.smithi145.stdout:found image23 2024-03-07T03:20:48.373 INFO:tasks.workunit.client.0.smithi145.stdout:erased image23 2024-03-07T03:20:48.373 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image24 2024-03-07T03:20:48.373 INFO:tasks.workunit.client.0.smithi145.stdout:found image24 2024-03-07T03:20:48.373 INFO:tasks.workunit.client.0.smithi145.stdout:erased image24 2024-03-07T03:20:48.373 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCopy (4861 ms) 2024-03-07T03:20:48.373 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCopyPP 2024-03-07T03:20:48.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:48 smithi145 bash[79878]: cluster 2024-03-07T03:20:47.371433+0000 mon.a (mon.0) 790 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:20:48.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:48 smithi145 bash[79878]: audit 2024-03-07T03:20:47.640378+0000 mon.a (mon.0) 791 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:48.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:48 smithi145 bash[82329]: cluster 2024-03-07T03:20:47.371433+0000 mon.a (mon.0) 790 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:20:48.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:48 smithi145 bash[82329]: audit 2024-03-07T03:20:47.640378+0000 mon.a (mon.0) 791 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:48.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:48 smithi189 bash[66969]: cluster 2024-03-07T03:20:47.371433+0000 mon.a (mon.0) 790 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:20:48.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:48 smithi189 bash[66969]: audit 2024-03-07T03:20:47.640378+0000 mon.a (mon.0) 791 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:49.189 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:20:49.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:49 smithi145 bash[79878]: cluster 2024-03-07T03:20:48.659157+0000 mgr.y (mgr.34107) 249 : cluster 0 pgmap v480: 201 pgs: 201 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 272 KiB/s rd, 19 KiB/s wr, 323 op/s 2024-03-07T03:20:49.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:49 smithi145 bash[82329]: cluster 2024-03-07T03:20:48.659157+0000 mgr.y (mgr.34107) 249 : cluster 0 pgmap v480: 201 pgs: 201 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 272 KiB/s rd, 19 KiB/s wr, 323 op/s 2024-03-07T03:20:49.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:49 smithi189 bash[66969]: cluster 2024-03-07T03:20:48.659157+0000 mgr.y (mgr.34107) 249 : cluster 0 pgmap v480: 201 pgs: 201 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 272 KiB/s rd, 19 KiB/s wr, 323 op/s 2024-03-07T03:20:50.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:49 smithi145 bash[72238]: debug 2024-03-07T03:20:49.707+0000 7f318c55a700 -1 mgr.server reply reply (11) Resource temporarily unavailable 32 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:20:50.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:49 smithi145 bash[72238]: debug 2024-03-07T03:20:49.707+0000 7f318c55a700 -1 mgr.server reply reply (16) Device or resource busy 32 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (0 PGs are or would become offline) 2024-03-07T03:20:50.422 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:50.806 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:50 smithi145 bash[79878]: cluster 2024-03-07T03:20:49.420101+0000 mon.a (mon.0) 792 : cluster 0 osdmap e587: 8 total, 8 up, 8 in 2024-03-07T03:20:50.806 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:50 smithi145 bash[79878]: audit 2024-03-07T03:20:49.422125+0000 mon.a (mon.0) 793 : audit 1 from='client.? 172.21.15.145:0/2341570566' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:50.806 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:50 smithi145 bash[79878]: audit 2024-03-07T03:20:49.710872+0000 mon.a (mon.0) 794 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:20:50.806 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:50 smithi145 bash[79878]: audit 2024-03-07T03:20:49.711342+0000 mgr.y (mgr.34107) 250 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:20:50.806 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:50 smithi145 bash[79878]: cephadm 2024-03-07T03:20:49.713803+0000 mgr.y (mgr.34107) 251 : cephadm 1 Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:20:50.807 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:50 smithi145 bash[82329]: cluster 2024-03-07T03:20:49.420101+0000 mon.a (mon.0) 792 : cluster 0 osdmap e587: 8 total, 8 up, 8 in 2024-03-07T03:20:50.807 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:50 smithi145 bash[82329]: audit 2024-03-07T03:20:49.422125+0000 mon.a (mon.0) 793 : audit 1 from='client.? 172.21.15.145:0/2341570566' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:50.807 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:50 smithi145 bash[82329]: audit 2024-03-07T03:20:49.710872+0000 mon.a (mon.0) 794 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:20:50.807 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:50 smithi145 bash[82329]: audit 2024-03-07T03:20:49.711342+0000 mgr.y (mgr.34107) 250 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:20:50.807 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:50 smithi145 bash[82329]: cephadm 2024-03-07T03:20:49.713803+0000 mgr.y (mgr.34107) 251 : cephadm 1 Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:20:50.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:50 smithi189 bash[66969]: cluster 2024-03-07T03:20:49.420101+0000 mon.a (mon.0) 792 : cluster 0 osdmap e587: 8 total, 8 up, 8 in 2024-03-07T03:20:50.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:50 smithi189 bash[66969]: audit 2024-03-07T03:20:49.422125+0000 mon.a (mon.0) 793 : audit 1 from='client.? 172.21.15.145:0/2341570566' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:50.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:50 smithi189 bash[66969]: audit 2024-03-07T03:20:49.710872+0000 mon.a (mon.0) 794 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:20:50.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:50 smithi189 bash[66969]: audit 2024-03-07T03:20:49.711342+0000 mgr.y (mgr.34107) 250 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:20:50.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:50 smithi189 bash[66969]: cephadm 2024-03-07T03:20:49.713803+0000 mgr.y (mgr.34107) 251 : cephadm 1 Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:20:51.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:51 smithi145 bash[82329]: audit 2024-03-07T03:20:50.411284+0000 mon.a (mon.0) 795 : audit 1 from='client.? 172.21.15.145:0/2341570566' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:51.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:51 smithi145 bash[82329]: cluster 2024-03-07T03:20:50.417233+0000 mon.a (mon.0) 796 : cluster 0 osdmap e588: 8 total, 8 up, 8 in 2024-03-07T03:20:51.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:51 smithi145 bash[82329]: cluster 2024-03-07T03:20:50.659801+0000 mgr.y (mgr.34107) 252 : cluster 0 pgmap v483: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 273 KiB/s rd, 19 KiB/s wr, 324 op/s 2024-03-07T03:20:51.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:51 smithi145 bash[79878]: audit 2024-03-07T03:20:50.411284+0000 mon.a (mon.0) 795 : audit 1 from='client.? 172.21.15.145:0/2341570566' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:51.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:51 smithi145 bash[79878]: cluster 2024-03-07T03:20:50.417233+0000 mon.a (mon.0) 796 : cluster 0 osdmap e588: 8 total, 8 up, 8 in 2024-03-07T03:20:51.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:51 smithi145 bash[79878]: cluster 2024-03-07T03:20:50.659801+0000 mgr.y (mgr.34107) 252 : cluster 0 pgmap v483: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 273 KiB/s rd, 19 KiB/s wr, 324 op/s 2024-03-07T03:20:51.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:51 smithi189 bash[66969]: audit 2024-03-07T03:20:50.411284+0000 mon.a (mon.0) 795 : audit 1 from='client.? 172.21.15.145:0/2341570566' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:51.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:51 smithi189 bash[66969]: cluster 2024-03-07T03:20:50.417233+0000 mon.a (mon.0) 796 : cluster 0 osdmap e588: 8 total, 8 up, 8 in 2024-03-07T03:20:51.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:51 smithi189 bash[66969]: cluster 2024-03-07T03:20:50.659801+0000 mgr.y (mgr.34107) 252 : cluster 0 pgmap v483: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 273 KiB/s rd, 19 KiB/s wr, 324 op/s 2024-03-07T03:20:52.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:52 smithi189 bash[66969]: cluster 2024-03-07T03:20:51.439432+0000 mon.a (mon.0) 797 : cluster 0 osdmap e589: 8 total, 8 up, 8 in 2024-03-07T03:20:52.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:52 smithi189 bash[66969]: cluster 2024-03-07T03:20:52.348544+0000 mon.a (mon.0) 798 : cluster 0 osdmap e590: 8 total, 8 up, 8 in 2024-03-07T03:20:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:20:52 smithi189 bash[59814]: ts=2024-03-07T03:20:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:52.749 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:52 smithi145 bash[82329]: cluster 2024-03-07T03:20:51.439432+0000 mon.a (mon.0) 797 : cluster 0 osdmap e589: 8 total, 8 up, 8 in 2024-03-07T03:20:52.749 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:52 smithi145 bash[82329]: cluster 2024-03-07T03:20:52.348544+0000 mon.a (mon.0) 798 : cluster 0 osdmap e590: 8 total, 8 up, 8 in 2024-03-07T03:20:52.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:52 smithi145 bash[79878]: cluster 2024-03-07T03:20:51.439432+0000 mon.a (mon.0) 797 : cluster 0 osdmap e589: 8 total, 8 up, 8 in 2024-03-07T03:20:52.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:52 smithi145 bash[79878]: cluster 2024-03-07T03:20:52.348544+0000 mon.a (mon.0) 798 : cluster 0 osdmap e590: 8 total, 8 up, 8 in 2024-03-07T03:20:52.951 INFO:tasks.workunit.client.0.smithi145.stdout:num images is: 1 2024-03-07T03:20:52.951 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:20:52.951 INFO:tasks.workunit.client.0.smithi145.stdout:image: image25 2024-03-07T03:20:52.952 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image25 2024-03-07T03:20:52.992 INFO:tasks.workunit.client.0.smithi145.stdout:num images is: 2 2024-03-07T03:20:52.992 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 2 2024-03-07T03:20:52.992 INFO:tasks.workunit.client.0.smithi145.stdout:image: image25 2024-03-07T03:20:52.992 INFO:tasks.workunit.client.0.smithi145.stdout:image: image26 2024-03-07T03:20:52.992 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image25 2024-03-07T03:20:52.992 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image26 2024-03-07T03:20:53.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:20:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:20:52] "GET /metrics HTTP/1.1" 200 43615 "" "Prometheus/2.43.0" 2024-03-07T03:20:53.084 INFO:tasks.workunit.client.0.smithi145.stdout:100.00% done 2024-03-07T03:20:53.085 INFO:tasks.workunit.client.0.smithi145.stdout:num images is: 3 2024-03-07T03:20:53.085 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 3 2024-03-07T03:20:53.085 INFO:tasks.workunit.client.0.smithi145.stdout:image: image25 2024-03-07T03:20:53.085 INFO:tasks.workunit.client.0.smithi145.stdout:image: image26 2024-03-07T03:20:53.085 INFO:tasks.workunit.client.0.smithi145.stdout:image: image27 2024-03-07T03:20:53.085 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image25 2024-03-07T03:20:53.085 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image26 2024-03-07T03:20:53.085 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image27 2024-03-07T03:20:53.184 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCopyPP (4813 ms) 2024-03-07T03:20:53.184 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestDeepCopy 2024-03-07T03:20:54.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:53 smithi145 bash[79878]: cluster 2024-03-07T03:20:52.660220+0000 mgr.y (mgr.34107) 253 : cluster 0 pgmap v486: 233 pgs: 20 unknown, 213 active+clean; 583 KiB data, 454 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 2.5 KiB/s wr, 57 op/s 2024-03-07T03:20:54.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:53 smithi145 bash[79878]: audit 2024-03-07T03:20:52.874330+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.145:0/3059291860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:20:54.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:53 smithi145 bash[82329]: cluster 2024-03-07T03:20:52.660220+0000 mgr.y (mgr.34107) 253 : cluster 0 pgmap v486: 233 pgs: 20 unknown, 213 active+clean; 583 KiB data, 454 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 2.5 KiB/s wr, 57 op/s 2024-03-07T03:20:54.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:53 smithi145 bash[82329]: audit 2024-03-07T03:20:52.874330+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.145:0/3059291860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:20:54.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:53 smithi189 bash[66969]: cluster 2024-03-07T03:20:52.660220+0000 mgr.y (mgr.34107) 253 : cluster 0 pgmap v486: 233 pgs: 20 unknown, 213 active+clean; 583 KiB data, 454 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 2.5 KiB/s wr, 57 op/s 2024-03-07T03:20:54.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:53 smithi189 bash[66969]: audit 2024-03-07T03:20:52.874330+0000 mon.c (mon.2) 11 : audit 0 from='client.? 172.21.15.145:0/3059291860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:20:54.737 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:20:55.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:54 smithi145 bash[79878]: cluster 2024-03-07T03:20:53.728306+0000 mon.a (mon.0) 799 : cluster 0 osdmap e591: 8 total, 8 up, 8 in 2024-03-07T03:20:55.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:54 smithi145 bash[79878]: audit 2024-03-07T03:20:53.729550+0000 mon.c (mon.2) 12 : audit 1 from='client.? 172.21.15.145:0/1238621721' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:55.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:54 smithi145 bash[79878]: audit 2024-03-07T03:20:53.730127+0000 mon.a (mon.0) 800 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:55.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:54 smithi145 bash[82329]: cluster 2024-03-07T03:20:53.728306+0000 mon.a (mon.0) 799 : cluster 0 osdmap e591: 8 total, 8 up, 8 in 2024-03-07T03:20:55.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:54 smithi145 bash[82329]: audit 2024-03-07T03:20:53.729550+0000 mon.c (mon.2) 12 : audit 1 from='client.? 172.21.15.145:0/1238621721' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:55.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:54 smithi145 bash[82329]: audit 2024-03-07T03:20:53.730127+0000 mon.a (mon.0) 800 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:55.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:54 smithi189 bash[66969]: cluster 2024-03-07T03:20:53.728306+0000 mon.a (mon.0) 799 : cluster 0 osdmap e591: 8 total, 8 up, 8 in 2024-03-07T03:20:55.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:54 smithi189 bash[66969]: audit 2024-03-07T03:20:53.729550+0000 mon.c (mon.2) 12 : audit 1 from='client.? 172.21.15.145:0/1238621721' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:55.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:54 smithi189 bash[66969]: audit 2024-03-07T03:20:53.730127+0000 mon.a (mon.0) 800 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:56.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:55 smithi145 bash[79878]: cluster 2024-03-07T03:20:54.661059+0000 mgr.y (mgr.34107) 254 : cluster 0 pgmap v488: 265 pgs: 32 unknown, 233 active+clean; 583 KiB data, 455 MiB used, 715 GiB / 715 GiB avail; 352 KiB/s rd, 22 KiB/s wr, 408 op/s 2024-03-07T03:20:56.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:55 smithi145 bash[79878]: cluster 2024-03-07T03:20:54.723009+0000 mon.a (mon.0) 801 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:20:56.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:55 smithi145 bash[79878]: audit 2024-03-07T03:20:54.732659+0000 mon.a (mon.0) 802 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:56.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:55 smithi145 bash[79878]: cluster 2024-03-07T03:20:54.742476+0000 mon.a (mon.0) 803 : cluster 0 osdmap e592: 8 total, 8 up, 8 in 2024-03-07T03:20:56.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:55 smithi145 bash[79878]: audit 2024-03-07T03:20:55.230277+0000 mon.a (mon.0) 804 : audit 0 from='client.? 172.21.15.145:0/925695583' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:20:56.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:55 smithi145 bash[82329]: cluster 2024-03-07T03:20:54.661059+0000 mgr.y (mgr.34107) 254 : cluster 0 pgmap v488: 265 pgs: 32 unknown, 233 active+clean; 583 KiB data, 455 MiB used, 715 GiB / 715 GiB avail; 352 KiB/s rd, 22 KiB/s wr, 408 op/s 2024-03-07T03:20:56.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:55 smithi145 bash[82329]: cluster 2024-03-07T03:20:54.723009+0000 mon.a (mon.0) 801 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:20:56.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:55 smithi145 bash[82329]: audit 2024-03-07T03:20:54.732659+0000 mon.a (mon.0) 802 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:56.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:55 smithi145 bash[82329]: cluster 2024-03-07T03:20:54.742476+0000 mon.a (mon.0) 803 : cluster 0 osdmap e592: 8 total, 8 up, 8 in 2024-03-07T03:20:56.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:55 smithi145 bash[82329]: audit 2024-03-07T03:20:55.230277+0000 mon.a (mon.0) 804 : audit 0 from='client.? 172.21.15.145:0/925695583' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:20:56.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:55 smithi189 bash[66969]: cluster 2024-03-07T03:20:54.661059+0000 mgr.y (mgr.34107) 254 : cluster 0 pgmap v488: 265 pgs: 32 unknown, 233 active+clean; 583 KiB data, 455 MiB used, 715 GiB / 715 GiB avail; 352 KiB/s rd, 22 KiB/s wr, 408 op/s 2024-03-07T03:20:56.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:55 smithi189 bash[66969]: cluster 2024-03-07T03:20:54.723009+0000 mon.a (mon.0) 801 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:20:56.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:55 smithi189 bash[66969]: audit 2024-03-07T03:20:54.732659+0000 mon.a (mon.0) 802 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:56.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:55 smithi189 bash[66969]: cluster 2024-03-07T03:20:54.742476+0000 mon.a (mon.0) 803 : cluster 0 osdmap e592: 8 total, 8 up, 8 in 2024-03-07T03:20:56.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:55 smithi189 bash[66969]: audit 2024-03-07T03:20:55.230277+0000 mon.a (mon.0) 804 : audit 0 from='client.? 172.21.15.145:0/925695583' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:20:57.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:56 smithi145 bash[79878]: cluster 2024-03-07T03:20:55.742527+0000 mon.a (mon.0) 805 : cluster 0 osdmap e593: 8 total, 8 up, 8 in 2024-03-07T03:20:57.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:56 smithi145 bash[82329]: cluster 2024-03-07T03:20:55.742527+0000 mon.a (mon.0) 805 : cluster 0 osdmap e593: 8 total, 8 up, 8 in 2024-03-07T03:20:57.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:56 smithi189 bash[66969]: cluster 2024-03-07T03:20:55.742527+0000 mon.a (mon.0) 805 : cluster 0 osdmap e593: 8 total, 8 up, 8 in 2024-03-07T03:20:58.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:57 smithi145 bash[79878]: cluster 2024-03-07T03:20:56.662314+0000 mgr.y (mgr.34107) 255 : cluster 0 pgmap v491: 265 pgs: 27 unknown, 238 active+clean; 583 KiB data, 455 MiB used, 715 GiB / 715 GiB avail; 298 KiB/s rd, 20 KiB/s wr, 350 op/s 2024-03-07T03:20:58.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:57 smithi145 bash[79878]: cluster 2024-03-07T03:20:56.747018+0000 mon.a (mon.0) 806 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:20:58.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:57 smithi145 bash[79878]: cluster 2024-03-07T03:20:56.768996+0000 mon.a (mon.0) 807 : cluster 0 osdmap e594: 8 total, 8 up, 8 in 2024-03-07T03:20:58.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:57 smithi145 bash[79878]: cluster 2024-03-07T03:20:57.346809+0000 mon.a (mon.0) 808 : cluster 0 osdmap e595: 8 total, 8 up, 8 in 2024-03-07T03:20:58.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:57 smithi145 bash[79878]: audit 2024-03-07T03:20:57.628035+0000 mon.a (mon.0) 809 : audit 0 from='client.? 172.21.15.145:0/2858481031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:20:58.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:57 smithi145 bash[82329]: cluster 2024-03-07T03:20:56.662314+0000 mgr.y (mgr.34107) 255 : cluster 0 pgmap v491: 265 pgs: 27 unknown, 238 active+clean; 583 KiB data, 455 MiB used, 715 GiB / 715 GiB avail; 298 KiB/s rd, 20 KiB/s wr, 350 op/s 2024-03-07T03:20:58.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:57 smithi145 bash[82329]: cluster 2024-03-07T03:20:56.747018+0000 mon.a (mon.0) 806 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:20:58.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:57 smithi145 bash[82329]: cluster 2024-03-07T03:20:56.768996+0000 mon.a (mon.0) 807 : cluster 0 osdmap e594: 8 total, 8 up, 8 in 2024-03-07T03:20:58.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:57 smithi145 bash[82329]: cluster 2024-03-07T03:20:57.346809+0000 mon.a (mon.0) 808 : cluster 0 osdmap e595: 8 total, 8 up, 8 in 2024-03-07T03:20:58.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:57 smithi145 bash[82329]: audit 2024-03-07T03:20:57.628035+0000 mon.a (mon.0) 809 : audit 0 from='client.? 172.21.15.145:0/2858481031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:20:58.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:57 smithi189 bash[66969]: cluster 2024-03-07T03:20:56.662314+0000 mgr.y (mgr.34107) 255 : cluster 0 pgmap v491: 265 pgs: 27 unknown, 238 active+clean; 583 KiB data, 455 MiB used, 715 GiB / 715 GiB avail; 298 KiB/s rd, 20 KiB/s wr, 350 op/s 2024-03-07T03:20:58.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:57 smithi189 bash[66969]: cluster 2024-03-07T03:20:56.747018+0000 mon.a (mon.0) 806 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:20:58.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:57 smithi189 bash[66969]: cluster 2024-03-07T03:20:56.768996+0000 mon.a (mon.0) 807 : cluster 0 osdmap e594: 8 total, 8 up, 8 in 2024-03-07T03:20:58.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:57 smithi189 bash[66969]: cluster 2024-03-07T03:20:57.346809+0000 mon.a (mon.0) 808 : cluster 0 osdmap e595: 8 total, 8 up, 8 in 2024-03-07T03:20:58.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:57 smithi189 bash[66969]: audit 2024-03-07T03:20:57.628035+0000 mon.a (mon.0) 809 : audit 0 from='client.? 172.21.15.145:0/2858481031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:20:59.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:59 smithi145 bash[79878]: cluster 2024-03-07T03:20:58.352329+0000 mon.a (mon.0) 810 : cluster 0 osdmap e596: 8 total, 8 up, 8 in 2024-03-07T03:20:59.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:20:59 smithi145 bash[79878]: cluster 2024-03-07T03:20:58.664311+0000 mgr.y (mgr.34107) 256 : cluster 0 pgmap v495: 265 pgs: 265 active+clean; 583 KiB data, 456 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 7.4 KiB/s wr, 152 op/s 2024-03-07T03:20:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:59 smithi145 bash[82329]: cluster 2024-03-07T03:20:58.352329+0000 mon.a (mon.0) 810 : cluster 0 osdmap e596: 8 total, 8 up, 8 in 2024-03-07T03:20:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:20:59 smithi145 bash[82329]: cluster 2024-03-07T03:20:58.664311+0000 mgr.y (mgr.34107) 256 : cluster 0 pgmap v495: 265 pgs: 265 active+clean; 583 KiB data, 456 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 7.4 KiB/s wr, 152 op/s 2024-03-07T03:20:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:59 smithi189 bash[66969]: cluster 2024-03-07T03:20:58.352329+0000 mon.a (mon.0) 810 : cluster 0 osdmap e596: 8 total, 8 up, 8 in 2024-03-07T03:20:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:20:59 smithi189 bash[66969]: cluster 2024-03-07T03:20:58.664311+0000 mgr.y (mgr.34107) 256 : cluster 0 pgmap v495: 265 pgs: 265 active+clean; 583 KiB data, 456 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 7.4 KiB/s wr, 152 op/s 2024-03-07T03:21:00.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:00 smithi145 bash[79878]: cluster 2024-03-07T03:20:59.368600+0000 mon.a (mon.0) 811 : cluster 0 osdmap e597: 8 total, 8 up, 8 in 2024-03-07T03:21:00.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:00 smithi145 bash[79878]: audit 2024-03-07T03:20:59.981481+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.145:0/3264403683' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:00.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:00 smithi145 bash[82329]: cluster 2024-03-07T03:20:59.368600+0000 mon.a (mon.0) 811 : cluster 0 osdmap e597: 8 total, 8 up, 8 in 2024-03-07T03:21:00.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:00 smithi145 bash[82329]: audit 2024-03-07T03:20:59.981481+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.145:0/3264403683' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:00 smithi189 bash[66969]: cluster 2024-03-07T03:20:59.368600+0000 mon.a (mon.0) 811 : cluster 0 osdmap e597: 8 total, 8 up, 8 in 2024-03-07T03:21:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:00 smithi189 bash[66969]: audit 2024-03-07T03:20:59.981481+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.145:0/3264403683' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:01.572 INFO:tasks.workunit.client.0.smithi145.stdout:image: image28 2024-03-07T03:21:01.572 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image28 2024-03-07T03:21:01.572 INFO:tasks.workunit.client.0.smithi145.stdout:found image28 2024-03-07T03:21:01.572 INFO:tasks.workunit.client.0.smithi145.stdout:erased image28 2024-03-07T03:21:01.572 INFO:tasks.workunit.client.0.smithi145.stdout:image: image28 2024-03-07T03:21:01.572 INFO:tasks.workunit.client.0.smithi145.stdout:image: image29 2024-03-07T03:21:01.572 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image28 2024-03-07T03:21:01.573 INFO:tasks.workunit.client.0.smithi145.stdout:found image28 2024-03-07T03:21:01.573 INFO:tasks.workunit.client.0.smithi145.stdout:erased image28 2024-03-07T03:21:01.573 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image29 2024-03-07T03:21:01.573 INFO:tasks.workunit.client.0.smithi145.stdout:found image29 2024-03-07T03:21:01.573 INFO:tasks.workunit.client.0.smithi145.stdout:erased image29 2024-03-07T03:21:01.573 INFO:tasks.workunit.client.0.smithi145.stdout:100.00% done 2024-03-07T03:21:01.573 INFO:tasks.workunit.client.0.smithi145.stdout:image: image28 2024-03-07T03:21:01.573 INFO:tasks.workunit.client.0.smithi145.stdout:image: image29 2024-03-07T03:21:01.573 INFO:tasks.workunit.client.0.smithi145.stdout:image: image30 2024-03-07T03:21:01.573 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image28 2024-03-07T03:21:01.573 INFO:tasks.workunit.client.0.smithi145.stdout:found image28 2024-03-07T03:21:01.573 INFO:tasks.workunit.client.0.smithi145.stdout:erased image28 2024-03-07T03:21:01.573 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image29 2024-03-07T03:21:01.574 INFO:tasks.workunit.client.0.smithi145.stdout:found image29 2024-03-07T03:21:01.574 INFO:tasks.workunit.client.0.smithi145.stdout:erased image29 2024-03-07T03:21:01.574 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image30 2024-03-07T03:21:01.574 INFO:tasks.workunit.client.0.smithi145.stdout:found image30 2024-03-07T03:21:01.574 INFO:tasks.workunit.client.0.smithi145.stdout:erased image30 2024-03-07T03:21:01.574 INFO:tasks.workunit.client.0.smithi145.stdout:image: image28 2024-03-07T03:21:01.574 INFO:tasks.workunit.client.0.smithi145.stdout:image: image29 2024-03-07T03:21:01.574 INFO:tasks.workunit.client.0.smithi145.stdout:image: image30 2024-03-07T03:21:01.574 INFO:tasks.workunit.client.0.smithi145.stdout:image: image31 2024-03-07T03:21:01.574 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image28 2024-03-07T03:21:01.574 INFO:tasks.workunit.client.0.smithi145.stdout:found image28 2024-03-07T03:21:01.574 INFO:tasks.workunit.client.0.smithi145.stdout:erased image28 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image29 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:found image29 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:erased image29 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image30 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:found image30 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:erased image30 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image31 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:found image31 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:erased image31 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:image: image28 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:image: image29 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:image: image30 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:image: image31 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:image: image32 2024-03-07T03:21:01.575 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image28 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:found image28 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:erased image28 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image29 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:found image29 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:erased image29 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image30 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:found image30 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:erased image30 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image31 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:found image31 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:erased image31 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image32 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:found image32 2024-03-07T03:21:01.576 INFO:tasks.workunit.client.0.smithi145.stdout:erased image32 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:100.00% done 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:image: image28 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:image: image29 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:image: image30 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:image: image31 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:image: image32 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:image: image33 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image28 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:found image28 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:erased image28 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image29 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:found image29 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:erased image29 2024-03-07T03:21:01.577 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image30 2024-03-07T03:21:01.578 INFO:tasks.workunit.client.0.smithi145.stdout:found image30 2024-03-07T03:21:01.578 INFO:tasks.workunit.client.0.smithi145.stdout:erased image30 2024-03-07T03:21:01.578 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image31 2024-03-07T03:21:01.578 INFO:tasks.workunit.client.0.smithi145.stdout:found image31 2024-03-07T03:21:01.578 INFO:tasks.workunit.client.0.smithi145.stdout:erased image31 2024-03-07T03:21:01.578 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image32 2024-03-07T03:21:01.578 INFO:tasks.workunit.client.0.smithi145.stdout:found image32 2024-03-07T03:21:01.578 INFO:tasks.workunit.client.0.smithi145.stdout:erased image32 2024-03-07T03:21:01.578 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image33 2024-03-07T03:21:01.578 INFO:tasks.workunit.client.0.smithi145.stdout:found image33 2024-03-07T03:21:01.578 INFO:tasks.workunit.client.0.smithi145.stdout:erased image33 2024-03-07T03:21:01.578 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestDeepCopy (8388 ms) 2024-03-07T03:21:01.578 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestDeepCopyPP 2024-03-07T03:21:01.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:01 smithi145 bash[79878]: cluster 2024-03-07T03:21:00.371783+0000 mon.a (mon.0) 812 : cluster 0 osdmap e598: 8 total, 8 up, 8 in 2024-03-07T03:21:01.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:01 smithi145 bash[79878]: cluster 2024-03-07T03:21:00.665171+0000 mgr.y (mgr.34107) 257 : cluster 0 pgmap v498: 265 pgs: 265 active+clean; 583 KiB data, 456 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 6.9 KiB/s wr, 151 op/s 2024-03-07T03:21:01.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:01 smithi145 bash[82329]: cluster 2024-03-07T03:21:00.371783+0000 mon.a (mon.0) 812 : cluster 0 osdmap e598: 8 total, 8 up, 8 in 2024-03-07T03:21:01.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:01 smithi145 bash[82329]: cluster 2024-03-07T03:21:00.665171+0000 mgr.y (mgr.34107) 257 : cluster 0 pgmap v498: 265 pgs: 265 active+clean; 583 KiB data, 456 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 6.9 KiB/s wr, 151 op/s 2024-03-07T03:21:01.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:01 smithi189 bash[66969]: cluster 2024-03-07T03:21:00.371783+0000 mon.a (mon.0) 812 : cluster 0 osdmap e598: 8 total, 8 up, 8 in 2024-03-07T03:21:01.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:01 smithi189 bash[66969]: cluster 2024-03-07T03:21:00.665171+0000 mgr.y (mgr.34107) 257 : cluster 0 pgmap v498: 265 pgs: 265 active+clean; 583 KiB data, 456 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 6.9 KiB/s wr, 151 op/s 2024-03-07T03:21:02.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:21:02 smithi189 bash[59814]: ts=2024-03-07T03:21:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:21:02.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:02 smithi189 bash[66969]: cluster 2024-03-07T03:21:01.391670+0000 mon.a (mon.0) 813 : cluster 0 osdmap e599: 8 total, 8 up, 8 in 2024-03-07T03:21:02.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:02 smithi189 bash[66969]: audit 2024-03-07T03:21:02.327625+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.145:0/2400936007' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:02.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:02 smithi189 bash[66969]: cluster 2024-03-07T03:21:02.351860+0000 mon.a (mon.0) 814 : cluster 0 osdmap e600: 8 total, 8 up, 8 in 2024-03-07T03:21:02.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:02 smithi189 bash[66969]: audit 2024-03-07T03:21:02.353592+0000 mon.a (mon.0) 815 : audit 1 from='client.? 172.21.15.145:0/1229698836' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:02.791 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:02 smithi145 bash[79878]: cluster 2024-03-07T03:21:01.391670+0000 mon.a (mon.0) 813 : cluster 0 osdmap e599: 8 total, 8 up, 8 in 2024-03-07T03:21:02.791 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:02 smithi145 bash[79878]: audit 2024-03-07T03:21:02.327625+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.145:0/2400936007' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:02.792 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:02 smithi145 bash[79878]: cluster 2024-03-07T03:21:02.351860+0000 mon.a (mon.0) 814 : cluster 0 osdmap e600: 8 total, 8 up, 8 in 2024-03-07T03:21:02.792 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:02 smithi145 bash[79878]: audit 2024-03-07T03:21:02.353592+0000 mon.a (mon.0) 815 : audit 1 from='client.? 172.21.15.145:0/1229698836' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:02.792 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:02 smithi145 bash[82329]: cluster 2024-03-07T03:21:01.391670+0000 mon.a (mon.0) 813 : cluster 0 osdmap e599: 8 total, 8 up, 8 in 2024-03-07T03:21:02.792 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:02 smithi145 bash[82329]: audit 2024-03-07T03:21:02.327625+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.145:0/2400936007' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:02.792 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:02 smithi145 bash[82329]: cluster 2024-03-07T03:21:02.351860+0000 mon.a (mon.0) 814 : cluster 0 osdmap e600: 8 total, 8 up, 8 in 2024-03-07T03:21:02.792 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:02 smithi145 bash[82329]: audit 2024-03-07T03:21:02.353592+0000 mon.a (mon.0) 815 : audit 1 from='client.? 172.21.15.145:0/1229698836' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:03.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:21:02] "GET /metrics HTTP/1.1" 200 50811 "" "Prometheus/2.43.0" 2024-03-07T03:21:03.353 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:21:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:03 smithi145 bash[79878]: audit 2024-03-07T03:21:02.640884+0000 mon.a (mon.0) 816 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:03 smithi145 bash[79878]: cluster 2024-03-07T03:21:02.666181+0000 mgr.y (mgr.34107) 258 : cluster 0 pgmap v501: 297 pgs: 32 unknown, 265 active+clean; 584 KiB data, 456 MiB used, 715 GiB / 715 GiB avail; 371 KiB/s rd, 22 KiB/s wr, 438 op/s 2024-03-07T03:21:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:03 smithi145 bash[79878]: audit 2024-03-07T03:21:03.349093+0000 mon.a (mon.0) 817 : audit 1 from='client.? 172.21.15.145:0/1229698836' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:03 smithi145 bash[79878]: cluster 2024-03-07T03:21:03.357251+0000 mon.a (mon.0) 818 : cluster 0 osdmap e601: 8 total, 8 up, 8 in 2024-03-07T03:21:03.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:03 smithi145 bash[82329]: audit 2024-03-07T03:21:02.640884+0000 mon.a (mon.0) 816 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:03.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:03 smithi145 bash[82329]: cluster 2024-03-07T03:21:02.666181+0000 mgr.y (mgr.34107) 258 : cluster 0 pgmap v501: 297 pgs: 32 unknown, 265 active+clean; 584 KiB data, 456 MiB used, 715 GiB / 715 GiB avail; 371 KiB/s rd, 22 KiB/s wr, 438 op/s 2024-03-07T03:21:03.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:03 smithi145 bash[82329]: audit 2024-03-07T03:21:03.349093+0000 mon.a (mon.0) 817 : audit 1 from='client.? 172.21.15.145:0/1229698836' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:03.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:03 smithi145 bash[82329]: cluster 2024-03-07T03:21:03.357251+0000 mon.a (mon.0) 818 : cluster 0 osdmap e601: 8 total, 8 up, 8 in 2024-03-07T03:21:03.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:03 smithi189 bash[66969]: audit 2024-03-07T03:21:02.640884+0000 mon.a (mon.0) 816 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:03.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:03 smithi189 bash[66969]: cluster 2024-03-07T03:21:02.666181+0000 mgr.y (mgr.34107) 258 : cluster 0 pgmap v501: 297 pgs: 32 unknown, 265 active+clean; 584 KiB data, 456 MiB used, 715 GiB / 715 GiB avail; 371 KiB/s rd, 22 KiB/s wr, 438 op/s 2024-03-07T03:21:03.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:03 smithi189 bash[66969]: audit 2024-03-07T03:21:03.349093+0000 mon.a (mon.0) 817 : audit 1 from='client.? 172.21.15.145:0/1229698836' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:03.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:03 smithi189 bash[66969]: cluster 2024-03-07T03:21:03.357251+0000 mon.a (mon.0) 818 : cluster 0 osdmap e601: 8 total, 8 up, 8 in 2024-03-07T03:21:04.715 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:04 smithi145 bash[82329]: cluster 2024-03-07T03:21:03.376789+0000 mon.a (mon.0) 819 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:21:04.716 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:04 smithi145 bash[82329]: cluster 2024-03-07T03:21:04.363270+0000 mon.a (mon.0) 820 : cluster 0 osdmap e602: 8 total, 8 up, 8 in 2024-03-07T03:21:04.716 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:04 smithi145 bash[79878]: cluster 2024-03-07T03:21:03.376789+0000 mon.a (mon.0) 819 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:21:04.716 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:04 smithi145 bash[79878]: cluster 2024-03-07T03:21:04.363270+0000 mon.a (mon.0) 820 : cluster 0 osdmap e602: 8 total, 8 up, 8 in 2024-03-07T03:21:04.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:04 smithi189 bash[66969]: cluster 2024-03-07T03:21:03.376789+0000 mon.a (mon.0) 819 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:21:04.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:04 smithi189 bash[66969]: cluster 2024-03-07T03:21:04.363270+0000 mon.a (mon.0) 820 : cluster 0 osdmap e602: 8 total, 8 up, 8 in 2024-03-07T03:21:05.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:04 smithi145 bash[72238]: debug 2024-03-07T03:21:04.723+0000 7f318c55a700 -1 mgr.server reply reply (11) Resource temporarily unavailable 10 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:21:05.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:04 smithi145 bash[72238]: debug 2024-03-07T03:21:04.723+0000 7f318c55a700 -1 mgr.server reply reply (16) Device or resource busy 10 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (0 PGs are or would become offline) 2024-03-07T03:21:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:05 smithi145 bash[79878]: cluster 2024-03-07T03:21:04.667030+0000 mgr.y (mgr.34107) 259 : cluster 0 pgmap v504: 297 pgs: 10 unknown, 287 active+clean; 584 KiB data, 460 MiB used, 715 GiB / 715 GiB avail; 426 KiB/s rd, 26 KiB/s wr, 505 op/s 2024-03-07T03:21:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:05 smithi145 bash[79878]: audit 2024-03-07T03:21:04.691363+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.145:0/2320350465' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:05 smithi145 bash[79878]: audit 2024-03-07T03:21:04.726555+0000 mon.a (mon.0) 821 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:21:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:05 smithi145 bash[79878]: audit 2024-03-07T03:21:04.726891+0000 mgr.y (mgr.34107) 260 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:21:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:05 smithi145 bash[79878]: cephadm 2024-03-07T03:21:04.728575+0000 mgr.y (mgr.34107) 261 : cephadm 1 Upgrade: 10 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:21:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:05 smithi145 bash[79878]: cluster 2024-03-07T03:21:05.363602+0000 mon.a (mon.0) 822 : cluster 0 osdmap e603: 8 total, 8 up, 8 in 2024-03-07T03:21:05.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:05 smithi145 bash[82329]: cluster 2024-03-07T03:21:04.667030+0000 mgr.y (mgr.34107) 259 : cluster 0 pgmap v504: 297 pgs: 10 unknown, 287 active+clean; 584 KiB data, 460 MiB used, 715 GiB / 715 GiB avail; 426 KiB/s rd, 26 KiB/s wr, 505 op/s 2024-03-07T03:21:05.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:05 smithi145 bash[82329]: audit 2024-03-07T03:21:04.691363+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.145:0/2320350465' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:05.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:05 smithi145 bash[82329]: audit 2024-03-07T03:21:04.726555+0000 mon.a (mon.0) 821 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:21:05.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:05 smithi145 bash[82329]: audit 2024-03-07T03:21:04.726891+0000 mgr.y (mgr.34107) 260 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:21:05.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:05 smithi145 bash[82329]: cephadm 2024-03-07T03:21:04.728575+0000 mgr.y (mgr.34107) 261 : cephadm 1 Upgrade: 10 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:21:05.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:05 smithi145 bash[82329]: cluster 2024-03-07T03:21:05.363602+0000 mon.a (mon.0) 822 : cluster 0 osdmap e603: 8 total, 8 up, 8 in 2024-03-07T03:21:05.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:05 smithi189 bash[66969]: cluster 2024-03-07T03:21:04.667030+0000 mgr.y (mgr.34107) 259 : cluster 0 pgmap v504: 297 pgs: 10 unknown, 287 active+clean; 584 KiB data, 460 MiB used, 715 GiB / 715 GiB avail; 426 KiB/s rd, 26 KiB/s wr, 505 op/s 2024-03-07T03:21:05.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:05 smithi189 bash[66969]: audit 2024-03-07T03:21:04.691363+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.145:0/2320350465' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:05.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:05 smithi189 bash[66969]: audit 2024-03-07T03:21:04.726555+0000 mon.a (mon.0) 821 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:21:05.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:05 smithi189 bash[66969]: audit 2024-03-07T03:21:04.726891+0000 mgr.y (mgr.34107) 260 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:21:05.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:05 smithi189 bash[66969]: cephadm 2024-03-07T03:21:04.728575+0000 mgr.y (mgr.34107) 261 : cephadm 1 Upgrade: 10 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:21:05.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:05 smithi189 bash[66969]: cluster 2024-03-07T03:21:05.363602+0000 mon.a (mon.0) 822 : cluster 0 osdmap e603: 8 total, 8 up, 8 in 2024-03-07T03:21:06.052 INFO:tasks.workunit.client.0.smithi145.stdout:num images is: 1 2024-03-07T03:21:06.052 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:21:06.053 INFO:tasks.workunit.client.0.smithi145.stdout:image: image34 2024-03-07T03:21:06.053 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image34 2024-03-07T03:21:06.127 INFO:tasks.workunit.client.0.smithi145.stdout:num images is: 2 2024-03-07T03:21:06.127 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 2 2024-03-07T03:21:06.127 INFO:tasks.workunit.client.0.smithi145.stdout:image: image34 2024-03-07T03:21:06.127 INFO:tasks.workunit.client.0.smithi145.stdout:image: image35 2024-03-07T03:21:06.127 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image34 2024-03-07T03:21:06.127 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image35 2024-03-07T03:21:06.277 INFO:tasks.workunit.client.0.smithi145.stdout:100.00% done 2024-03-07T03:21:06.277 INFO:tasks.workunit.client.0.smithi145.stdout:num images is: 3 2024-03-07T03:21:06.277 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 3 2024-03-07T03:21:06.277 INFO:tasks.workunit.client.0.smithi145.stdout:image: image34 2024-03-07T03:21:06.277 INFO:tasks.workunit.client.0.smithi145.stdout:image: image35 2024-03-07T03:21:06.277 INFO:tasks.workunit.client.0.smithi145.stdout:image: image36 2024-03-07T03:21:06.277 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image34 2024-03-07T03:21:06.277 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image35 2024-03-07T03:21:06.277 INFO:tasks.workunit.client.0.smithi145.stdout:expected = image36 2024-03-07T03:21:06.397 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestDeepCopyPP (4825 ms) 2024-03-07T03:21:06.397 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnap 2024-03-07T03:21:06.397 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:21:06.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:06 smithi145 bash[82329]: cluster 2024-03-07T03:21:05.397209+0000 mon.a (mon.0) 823 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:21:06.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:06 smithi145 bash[79878]: cluster 2024-03-07T03:21:05.397209+0000 mon.a (mon.0) 823 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:21:06.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:06 smithi189 bash[66969]: cluster 2024-03-07T03:21:05.397209+0000 mon.a (mon.0) 823 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:21:07.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:07 smithi145 bash[79878]: cluster 2024-03-07T03:21:06.668269+0000 mgr.y (mgr.34107) 262 : cluster 0 pgmap v506: 297 pgs: 8 unknown, 289 active+clean; 584 KiB data, 460 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 4.4 KiB/s wr, 81 op/s 2024-03-07T03:21:07.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:07 smithi145 bash[79878]: audit 2024-03-07T03:21:07.044445+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.145:0/1887125374' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:07.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:07 smithi145 bash[79878]: cluster 2024-03-07T03:21:07.354006+0000 mon.a (mon.0) 824 : cluster 0 osdmap e604: 8 total, 8 up, 8 in 2024-03-07T03:21:07.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:07 smithi145 bash[82329]: cluster 2024-03-07T03:21:06.668269+0000 mgr.y (mgr.34107) 262 : cluster 0 pgmap v506: 297 pgs: 8 unknown, 289 active+clean; 584 KiB data, 460 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 4.4 KiB/s wr, 81 op/s 2024-03-07T03:21:07.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:07 smithi145 bash[82329]: audit 2024-03-07T03:21:07.044445+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.145:0/1887125374' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:07.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:07 smithi145 bash[82329]: cluster 2024-03-07T03:21:07.354006+0000 mon.a (mon.0) 824 : cluster 0 osdmap e604: 8 total, 8 up, 8 in 2024-03-07T03:21:07.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:07 smithi189 bash[66969]: cluster 2024-03-07T03:21:06.668269+0000 mgr.y (mgr.34107) 262 : cluster 0 pgmap v506: 297 pgs: 8 unknown, 289 active+clean; 584 KiB data, 460 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 4.4 KiB/s wr, 81 op/s 2024-03-07T03:21:07.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:07 smithi189 bash[66969]: audit 2024-03-07T03:21:07.044445+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.145:0/1887125374' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:07.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:07 smithi189 bash[66969]: cluster 2024-03-07T03:21:07.354006+0000 mon.a (mon.0) 824 : cluster 0 osdmap e604: 8 total, 8 up, 8 in 2024-03-07T03:21:09.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:09 smithi145 bash[79878]: cluster 2024-03-07T03:21:08.359072+0000 mon.a (mon.0) 825 : cluster 0 osdmap e605: 8 total, 8 up, 8 in 2024-03-07T03:21:09.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:09 smithi145 bash[79878]: cluster 2024-03-07T03:21:08.670515+0000 mgr.y (mgr.34107) 263 : cluster 0 pgmap v509: 297 pgs: 297 active+clean; 584 KiB data, 461 MiB used, 715 GiB / 715 GiB avail; 365 KiB/s rd, 24 KiB/s wr, 432 op/s 2024-03-07T03:21:09.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:09 smithi145 bash[82329]: cluster 2024-03-07T03:21:08.359072+0000 mon.a (mon.0) 825 : cluster 0 osdmap e605: 8 total, 8 up, 8 in 2024-03-07T03:21:09.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:09 smithi145 bash[82329]: cluster 2024-03-07T03:21:08.670515+0000 mgr.y (mgr.34107) 263 : cluster 0 pgmap v509: 297 pgs: 297 active+clean; 584 KiB data, 461 MiB used, 715 GiB / 715 GiB avail; 365 KiB/s rd, 24 KiB/s wr, 432 op/s 2024-03-07T03:21:09.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:09 smithi189 bash[66969]: cluster 2024-03-07T03:21:08.359072+0000 mon.a (mon.0) 825 : cluster 0 osdmap e605: 8 total, 8 up, 8 in 2024-03-07T03:21:09.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:09 smithi189 bash[66969]: cluster 2024-03-07T03:21:08.670515+0000 mgr.y (mgr.34107) 263 : cluster 0 pgmap v509: 297 pgs: 297 active+clean; 584 KiB data, 461 MiB used, 715 GiB / 715 GiB avail; 365 KiB/s rd, 24 KiB/s wr, 432 op/s 2024-03-07T03:21:10.419 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 1 2024-03-07T03:21:10.419 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:21:10.419 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap1 2024-03-07T03:21:10.420 INFO:tasks.workunit.client.0.smithi145.stdout:found snap1 with size 2097152 2024-03-07T03:21:10.420 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 2 2024-03-07T03:21:10.420 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 2 2024-03-07T03:21:10.420 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap1 2024-03-07T03:21:10.420 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap2 2024-03-07T03:21:10.420 INFO:tasks.workunit.client.0.smithi145.stdout:found snap1 with size 2097152 2024-03-07T03:21:10.420 INFO:tasks.workunit.client.0.smithi145.stdout:found snap2 with size 4194304 2024-03-07T03:21:10.420 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 1 2024-03-07T03:21:10.420 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:21:10.420 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap2 2024-03-07T03:21:10.420 INFO:tasks.workunit.client.0.smithi145.stdout:found snap2 with size 4194304 2024-03-07T03:21:10.420 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 0 2024-03-07T03:21:10.420 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 0 2024-03-07T03:21:10.421 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnap (4022 ms) 2024-03-07T03:21:10.421 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestGetSnapShotTimeStamp 2024-03-07T03:21:10.421 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:21:10.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:10 smithi145 bash[79878]: cluster 2024-03-07T03:21:09.369743+0000 mon.a (mon.0) 826 : cluster 0 osdmap e606: 8 total, 8 up, 8 in 2024-03-07T03:21:10.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:10 smithi145 bash[79878]: audit 2024-03-07T03:21:09.420121+0000 mon.a (mon.0) 827 : audit 0 from='client.? 172.21.15.145:0/936319352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:10.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:10 smithi145 bash[82329]: cluster 2024-03-07T03:21:09.369743+0000 mon.a (mon.0) 826 : cluster 0 osdmap e606: 8 total, 8 up, 8 in 2024-03-07T03:21:10.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:10 smithi145 bash[82329]: audit 2024-03-07T03:21:09.420121+0000 mon.a (mon.0) 827 : audit 0 from='client.? 172.21.15.145:0/936319352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:10.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:10 smithi189 bash[66969]: cluster 2024-03-07T03:21:09.369743+0000 mon.a (mon.0) 826 : cluster 0 osdmap e606: 8 total, 8 up, 8 in 2024-03-07T03:21:10.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:10 smithi189 bash[66969]: audit 2024-03-07T03:21:09.420121+0000 mon.a (mon.0) 827 : audit 0 from='client.? 172.21.15.145:0/936319352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:11.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:11 smithi145 bash[79878]: cluster 2024-03-07T03:21:10.376882+0000 mon.a (mon.0) 828 : cluster 0 osdmap e607: 8 total, 8 up, 8 in 2024-03-07T03:21:11.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:11 smithi145 bash[79878]: cluster 2024-03-07T03:21:10.671387+0000 mgr.y (mgr.34107) 264 : cluster 0 pgmap v512: 297 pgs: 297 active+clean; 584 KiB data, 461 MiB used, 715 GiB / 715 GiB avail; 373 KiB/s rd, 25 KiB/s wr, 444 op/s 2024-03-07T03:21:11.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:11 smithi145 bash[82329]: cluster 2024-03-07T03:21:10.376882+0000 mon.a (mon.0) 828 : cluster 0 osdmap e607: 8 total, 8 up, 8 in 2024-03-07T03:21:11.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:11 smithi145 bash[82329]: cluster 2024-03-07T03:21:10.671387+0000 mgr.y (mgr.34107) 264 : cluster 0 pgmap v512: 297 pgs: 297 active+clean; 584 KiB data, 461 MiB used, 715 GiB / 715 GiB avail; 373 KiB/s rd, 25 KiB/s wr, 444 op/s 2024-03-07T03:21:11.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:11 smithi189 bash[66969]: cluster 2024-03-07T03:21:10.376882+0000 mon.a (mon.0) 828 : cluster 0 osdmap e607: 8 total, 8 up, 8 in 2024-03-07T03:21:11.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:11 smithi189 bash[66969]: cluster 2024-03-07T03:21:10.671387+0000 mgr.y (mgr.34107) 264 : cluster 0 pgmap v512: 297 pgs: 297 active+clean; 584 KiB data, 461 MiB used, 715 GiB / 715 GiB avail; 373 KiB/s rd, 25 KiB/s wr, 444 op/s 2024-03-07T03:21:12.417 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestGetSnapShotTimeStamp (1998 ms) 2024-03-07T03:21:12.417 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnapPP 2024-03-07T03:21:12.417 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:21:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:21:12 smithi189 bash[59814]: ts=2024-03-07T03:21:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:21:12.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:12 smithi189 bash[66969]: cluster 2024-03-07T03:21:11.406505+0000 mon.a (mon.0) 829 : cluster 0 osdmap e608: 8 total, 8 up, 8 in 2024-03-07T03:21:12.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:12 smithi189 bash[66969]: audit 2024-03-07T03:21:11.774230+0000 mon.a (mon.0) 830 : audit 0 from='client.? 172.21.15.145:0/48611983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:12.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:12 smithi189 bash[66969]: cluster 2024-03-07T03:21:12.360534+0000 mon.a (mon.0) 831 : cluster 0 osdmap e609: 8 total, 8 up, 8 in 2024-03-07T03:21:12.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:12 smithi145 bash[82329]: cluster 2024-03-07T03:21:11.406505+0000 mon.a (mon.0) 829 : cluster 0 osdmap e608: 8 total, 8 up, 8 in 2024-03-07T03:21:12.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:12 smithi145 bash[82329]: audit 2024-03-07T03:21:11.774230+0000 mon.a (mon.0) 830 : audit 0 from='client.? 172.21.15.145:0/48611983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:12.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:12 smithi145 bash[82329]: cluster 2024-03-07T03:21:12.360534+0000 mon.a (mon.0) 831 : cluster 0 osdmap e609: 8 total, 8 up, 8 in 2024-03-07T03:21:12.752 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:12 smithi145 bash[79878]: cluster 2024-03-07T03:21:11.406505+0000 mon.a (mon.0) 829 : cluster 0 osdmap e608: 8 total, 8 up, 8 in 2024-03-07T03:21:12.752 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:12 smithi145 bash[79878]: audit 2024-03-07T03:21:11.774230+0000 mon.a (mon.0) 830 : audit 0 from='client.? 172.21.15.145:0/48611983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:12.752 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:12 smithi145 bash[79878]: cluster 2024-03-07T03:21:12.360534+0000 mon.a (mon.0) 831 : cluster 0 osdmap e609: 8 total, 8 up, 8 in 2024-03-07T03:21:13.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:21:12] "GET /metrics HTTP/1.1" 200 50811 "" "Prometheus/2.43.0" 2024-03-07T03:21:13.447 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 1 2024-03-07T03:21:13.447 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:21:13.447 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap1 2024-03-07T03:21:13.447 INFO:tasks.workunit.client.0.smithi145.stdout:found snap1 with size 2097152 2024-03-07T03:21:13.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:13 smithi145 bash[79878]: cluster 2024-03-07T03:21:12.672257+0000 mgr.y (mgr.34107) 265 : cluster 0 pgmap v515: 297 pgs: 297 active+clean; 584 KiB data, 461 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 767 B/s wr, 14 op/s 2024-03-07T03:21:13.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:13 smithi145 bash[82329]: cluster 2024-03-07T03:21:12.672257+0000 mgr.y (mgr.34107) 265 : cluster 0 pgmap v515: 297 pgs: 297 active+clean; 584 KiB data, 461 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 767 B/s wr, 14 op/s 2024-03-07T03:21:13.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:13 smithi189 bash[66969]: cluster 2024-03-07T03:21:12.672257+0000 mgr.y (mgr.34107) 265 : cluster 0 pgmap v515: 297 pgs: 297 active+clean; 584 KiB data, 461 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 767 B/s wr, 14 op/s 2024-03-07T03:21:14.467 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 2 2024-03-07T03:21:14.467 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 2 2024-03-07T03:21:14.467 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap1 2024-03-07T03:21:14.467 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap2 2024-03-07T03:21:14.467 INFO:tasks.workunit.client.0.smithi145.stdout:found snap1 with size 2097152 2024-03-07T03:21:14.467 INFO:tasks.workunit.client.0.smithi145.stdout:found snap2 with size 4194304 2024-03-07T03:21:14.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:14 smithi145 bash[79878]: cluster 2024-03-07T03:21:13.418894+0000 mon.a (mon.0) 832 : cluster 0 osdmap e610: 8 total, 8 up, 8 in 2024-03-07T03:21:14.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:14 smithi145 bash[79878]: audit 2024-03-07T03:21:13.657864+0000 mon.a (mon.0) 833 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:14.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:14 smithi145 bash[79878]: audit 2024-03-07T03:21:14.128460+0000 mon.a (mon.0) 834 : audit 0 from='client.? 172.21.15.145:0/3699645596' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:14.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:14 smithi145 bash[82329]: cluster 2024-03-07T03:21:13.418894+0000 mon.a (mon.0) 832 : cluster 0 osdmap e610: 8 total, 8 up, 8 in 2024-03-07T03:21:14.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:14 smithi145 bash[82329]: audit 2024-03-07T03:21:13.657864+0000 mon.a (mon.0) 833 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:14.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:14 smithi145 bash[82329]: audit 2024-03-07T03:21:14.128460+0000 mon.a (mon.0) 834 : audit 0 from='client.? 172.21.15.145:0/3699645596' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:14.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:14 smithi189 bash[66969]: cluster 2024-03-07T03:21:13.418894+0000 mon.a (mon.0) 832 : cluster 0 osdmap e610: 8 total, 8 up, 8 in 2024-03-07T03:21:14.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:14 smithi189 bash[66969]: audit 2024-03-07T03:21:13.657864+0000 mon.a (mon.0) 833 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:14.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:14 smithi189 bash[66969]: audit 2024-03-07T03:21:14.128460+0000 mon.a (mon.0) 834 : audit 0 from='client.? 172.21.15.145:0/3699645596' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:15.462 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 1 2024-03-07T03:21:15.462 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:21:15.462 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap2 2024-03-07T03:21:15.463 INFO:tasks.workunit.client.0.smithi145.stdout:found snap2 with size 4194304 2024-03-07T03:21:15.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:15 smithi145 bash[82329]: cluster 2024-03-07T03:21:14.420831+0000 mon.a (mon.0) 835 : cluster 0 osdmap e611: 8 total, 8 up, 8 in 2024-03-07T03:21:15.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:15 smithi145 bash[82329]: cluster 2024-03-07T03:21:14.673262+0000 mgr.y (mgr.34107) 266 : cluster 0 pgmap v518: 297 pgs: 297 active+clean; 584 KiB data, 466 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s rd, 5.5 KiB/s wr, 69 op/s 2024-03-07T03:21:15.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:15 smithi145 bash[79878]: cluster 2024-03-07T03:21:14.420831+0000 mon.a (mon.0) 835 : cluster 0 osdmap e611: 8 total, 8 up, 8 in 2024-03-07T03:21:15.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:15 smithi145 bash[79878]: cluster 2024-03-07T03:21:14.673262+0000 mgr.y (mgr.34107) 266 : cluster 0 pgmap v518: 297 pgs: 297 active+clean; 584 KiB data, 466 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s rd, 5.5 KiB/s wr, 69 op/s 2024-03-07T03:21:15.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:15 smithi189 bash[66969]: cluster 2024-03-07T03:21:14.420831+0000 mon.a (mon.0) 835 : cluster 0 osdmap e611: 8 total, 8 up, 8 in 2024-03-07T03:21:15.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:15 smithi189 bash[66969]: cluster 2024-03-07T03:21:14.673262+0000 mgr.y (mgr.34107) 266 : cluster 0 pgmap v518: 297 pgs: 297 active+clean; 584 KiB data, 466 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s rd, 5.5 KiB/s wr, 69 op/s 2024-03-07T03:21:16.467 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 0 2024-03-07T03:21:16.467 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 0 2024-03-07T03:21:16.476 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnapPP (4059 ms) 2024-03-07T03:21:16.476 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestGetNameIdSnapPP 2024-03-07T03:21:16.476 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:21:16.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:16 smithi145 bash[79878]: cluster 2024-03-07T03:21:15.448341+0000 mon.a (mon.0) 836 : cluster 0 osdmap e612: 8 total, 8 up, 8 in 2024-03-07T03:21:16.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:16 smithi145 bash[82329]: cluster 2024-03-07T03:21:15.448341+0000 mon.a (mon.0) 836 : cluster 0 osdmap e612: 8 total, 8 up, 8 in 2024-03-07T03:21:16.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:16 smithi189 bash[66969]: cluster 2024-03-07T03:21:15.448341+0000 mon.a (mon.0) 836 : cluster 0 osdmap e612: 8 total, 8 up, 8 in 2024-03-07T03:21:17.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:17 smithi145 bash[79878]: cluster 2024-03-07T03:21:16.443045+0000 mon.a (mon.0) 837 : cluster 0 osdmap e613: 8 total, 8 up, 8 in 2024-03-07T03:21:17.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:17 smithi145 bash[79878]: audit 2024-03-07T03:21:16.492040+0000 mon.a (mon.0) 838 : audit 0 from='client.? 172.21.15.145:0/4018565897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:17.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:17 smithi145 bash[79878]: cluster 2024-03-07T03:21:16.674435+0000 mgr.y (mgr.34107) 267 : cluster 0 pgmap v521: 297 pgs: 297 active+clean; 584 KiB data, 466 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 5.5 KiB/s wr, 66 op/s 2024-03-07T03:21:17.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:17 smithi145 bash[79878]: cluster 2024-03-07T03:21:17.358811+0000 mon.a (mon.0) 839 : cluster 0 osdmap e614: 8 total, 8 up, 8 in 2024-03-07T03:21:17.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:17 smithi145 bash[82329]: cluster 2024-03-07T03:21:16.443045+0000 mon.a (mon.0) 837 : cluster 0 osdmap e613: 8 total, 8 up, 8 in 2024-03-07T03:21:17.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:17 smithi145 bash[82329]: audit 2024-03-07T03:21:16.492040+0000 mon.a (mon.0) 838 : audit 0 from='client.? 172.21.15.145:0/4018565897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:17.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:17 smithi145 bash[82329]: cluster 2024-03-07T03:21:16.674435+0000 mgr.y (mgr.34107) 267 : cluster 0 pgmap v521: 297 pgs: 297 active+clean; 584 KiB data, 466 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 5.5 KiB/s wr, 66 op/s 2024-03-07T03:21:17.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:17 smithi145 bash[82329]: cluster 2024-03-07T03:21:17.358811+0000 mon.a (mon.0) 839 : cluster 0 osdmap e614: 8 total, 8 up, 8 in 2024-03-07T03:21:17.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:17 smithi189 bash[66969]: cluster 2024-03-07T03:21:16.443045+0000 mon.a (mon.0) 837 : cluster 0 osdmap e613: 8 total, 8 up, 8 in 2024-03-07T03:21:17.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:17 smithi189 bash[66969]: audit 2024-03-07T03:21:16.492040+0000 mon.a (mon.0) 838 : audit 0 from='client.? 172.21.15.145:0/4018565897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:17.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:17 smithi189 bash[66969]: cluster 2024-03-07T03:21:16.674435+0000 mgr.y (mgr.34107) 267 : cluster 0 pgmap v521: 297 pgs: 297 active+clean; 584 KiB data, 466 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 5.5 KiB/s wr, 66 op/s 2024-03-07T03:21:17.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:17 smithi189 bash[66969]: cluster 2024-03-07T03:21:17.358811+0000 mon.a (mon.0) 839 : cluster 0 osdmap e614: 8 total, 8 up, 8 in 2024-03-07T03:21:18.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:18 smithi145 bash[79878]: audit 2024-03-07T03:21:17.641312+0000 mon.a (mon.0) 840 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:18.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:18 smithi145 bash[79878]: cluster 2024-03-07T03:21:18.361151+0000 mon.a (mon.0) 841 : cluster 0 osdmap e615: 8 total, 8 up, 8 in 2024-03-07T03:21:18.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:18 smithi145 bash[82329]: audit 2024-03-07T03:21:17.641312+0000 mon.a (mon.0) 840 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:18.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:18 smithi145 bash[82329]: cluster 2024-03-07T03:21:18.361151+0000 mon.a (mon.0) 841 : cluster 0 osdmap e615: 8 total, 8 up, 8 in 2024-03-07T03:21:18.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:18 smithi189 bash[66969]: audit 2024-03-07T03:21:17.641312+0000 mon.a (mon.0) 840 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:18.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:18 smithi189 bash[66969]: cluster 2024-03-07T03:21:18.361151+0000 mon.a (mon.0) 841 : cluster 0 osdmap e615: 8 total, 8 up, 8 in 2024-03-07T03:21:19.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:19 smithi145 bash[79878]: cluster 2024-03-07T03:21:18.675577+0000 mgr.y (mgr.34107) 268 : cluster 0 pgmap v524: 297 pgs: 297 active+clean; 584 KiB data, 470 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 5.7 KiB/s wr, 74 op/s 2024-03-07T03:21:19.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:19 smithi145 bash[79878]: audit 2024-03-07T03:21:18.860359+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.145:0/1858231387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:19.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:19 smithi145 bash[79878]: cluster 2024-03-07T03:21:19.366407+0000 mon.a (mon.0) 842 : cluster 0 osdmap e616: 8 total, 8 up, 8 in 2024-03-07T03:21:19.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:19 smithi145 bash[82329]: cluster 2024-03-07T03:21:18.675577+0000 mgr.y (mgr.34107) 268 : cluster 0 pgmap v524: 297 pgs: 297 active+clean; 584 KiB data, 470 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 5.7 KiB/s wr, 74 op/s 2024-03-07T03:21:19.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:19 smithi145 bash[82329]: audit 2024-03-07T03:21:18.860359+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.145:0/1858231387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:19.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:19 smithi145 bash[82329]: cluster 2024-03-07T03:21:19.366407+0000 mon.a (mon.0) 842 : cluster 0 osdmap e616: 8 total, 8 up, 8 in 2024-03-07T03:21:19.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:19 smithi189 bash[66969]: cluster 2024-03-07T03:21:18.675577+0000 mgr.y (mgr.34107) 268 : cluster 0 pgmap v524: 297 pgs: 297 active+clean; 584 KiB data, 470 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 5.7 KiB/s wr, 74 op/s 2024-03-07T03:21:19.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:19 smithi189 bash[66969]: audit 2024-03-07T03:21:18.860359+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.145:0/1858231387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:19.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:19 smithi189 bash[66969]: cluster 2024-03-07T03:21:19.366407+0000 mon.a (mon.0) 842 : cluster 0 osdmap e616: 8 total, 8 up, 8 in 2024-03-07T03:21:20.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:20 smithi145 bash[82329]: audit 2024-03-07T03:21:19.742930+0000 mon.a (mon.0) 843 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:21:20.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:20 smithi145 bash[82329]: audit 2024-03-07T03:21:19.743356+0000 mgr.y (mgr.34107) 269 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:21:20.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:20 smithi145 bash[82329]: cephadm 2024-03-07T03:21:19.746653+0000 mgr.y (mgr.34107) 270 : cephadm 1 Upgrade: osd.2 is safe to restart 2024-03-07T03:21:20.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:20 smithi145 bash[82329]: cluster 2024-03-07T03:21:20.367075+0000 mon.a (mon.0) 844 : cluster 0 osdmap e617: 8 total, 8 up, 8 in 2024-03-07T03:21:20.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:20 smithi145 bash[79878]: audit 2024-03-07T03:21:19.742930+0000 mon.a (mon.0) 843 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:21:20.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:20 smithi145 bash[79878]: audit 2024-03-07T03:21:19.743356+0000 mgr.y (mgr.34107) 269 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:21:20.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:20 smithi145 bash[79878]: cephadm 2024-03-07T03:21:19.746653+0000 mgr.y (mgr.34107) 270 : cephadm 1 Upgrade: osd.2 is safe to restart 2024-03-07T03:21:20.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:20 smithi145 bash[79878]: cluster 2024-03-07T03:21:20.367075+0000 mon.a (mon.0) 844 : cluster 0 osdmap e617: 8 total, 8 up, 8 in 2024-03-07T03:21:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:20 smithi189 bash[66969]: audit 2024-03-07T03:21:19.742930+0000 mon.a (mon.0) 843 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:21:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:20 smithi189 bash[66969]: audit 2024-03-07T03:21:19.743356+0000 mgr.y (mgr.34107) 269 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:21:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:20 smithi189 bash[66969]: cephadm 2024-03-07T03:21:19.746653+0000 mgr.y (mgr.34107) 270 : cephadm 1 Upgrade: osd.2 is safe to restart 2024-03-07T03:21:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:20 smithi189 bash[66969]: cluster 2024-03-07T03:21:20.367075+0000 mon.a (mon.0) 844 : cluster 0 osdmap e617: 8 total, 8 up, 8 in 2024-03-07T03:21:21.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:21 smithi145 bash[79878]: cluster 2024-03-07T03:21:20.676401+0000 mgr.y (mgr.34107) 271 : cluster 0 pgmap v527: 297 pgs: 297 active+clean; 584 KiB data, 470 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 5.0 KiB/s wr, 62 op/s 2024-03-07T03:21:21.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:21 smithi145 bash[79878]: audit 2024-03-07T03:21:21.224853+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.145:0/1967446282' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:21.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:21 smithi145 bash[79878]: cluster 2024-03-07T03:21:21.370590+0000 mon.a (mon.0) 845 : cluster 0 osdmap e618: 8 total, 8 up, 8 in 2024-03-07T03:21:21.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:21 smithi145 bash[82329]: cluster 2024-03-07T03:21:20.676401+0000 mgr.y (mgr.34107) 271 : cluster 0 pgmap v527: 297 pgs: 297 active+clean; 584 KiB data, 470 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 5.0 KiB/s wr, 62 op/s 2024-03-07T03:21:21.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:21 smithi145 bash[82329]: audit 2024-03-07T03:21:21.224853+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.145:0/1967446282' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:21.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:21 smithi145 bash[82329]: cluster 2024-03-07T03:21:21.370590+0000 mon.a (mon.0) 845 : cluster 0 osdmap e618: 8 total, 8 up, 8 in 2024-03-07T03:21:21.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:21 smithi189 bash[66969]: cluster 2024-03-07T03:21:20.676401+0000 mgr.y (mgr.34107) 271 : cluster 0 pgmap v527: 297 pgs: 297 active+clean; 584 KiB data, 470 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 5.0 KiB/s wr, 62 op/s 2024-03-07T03:21:21.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:21 smithi189 bash[66969]: audit 2024-03-07T03:21:21.224853+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.145:0/1967446282' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:21.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:21 smithi189 bash[66969]: cluster 2024-03-07T03:21:21.370590+0000 mon.a (mon.0) 845 : cluster 0 osdmap e618: 8 total, 8 up, 8 in 2024-03-07T03:21:22.379 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 0 2024-03-07T03:21:22.379 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 0 2024-03-07T03:21:22.393 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestGetNameIdSnapPP (5917 ms) 2024-03-07T03:21:22.393 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCreateLsRenameSnapPP 2024-03-07T03:21:22.393 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:21:22.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:21:22 smithi189 bash[59814]: ts=2024-03-07T03:21:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:21:23.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:21:22] "GET /metrics HTTP/1.1" 200 50866 "" "Prometheus/2.43.0" 2024-03-07T03:21:23.389 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 1 2024-03-07T03:21:23.389 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:21:23.389 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap1 2024-03-07T03:21:23.389 INFO:tasks.workunit.client.0.smithi145.stdout:found snap1 with size 2097152 2024-03-07T03:21:23.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[79878]: cluster 2024-03-07T03:21:22.362468+0000 mon.a (mon.0) 846 : cluster 0 osdmap e619: 8 total, 8 up, 8 in 2024-03-07T03:21:23.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[79878]: cephadm 2024-03-07T03:21:22.472313+0000 mgr.y (mgr.34107) 272 : cephadm 1 Upgrade: Updating osd.2 2024-03-07T03:21:23.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[79878]: audit 2024-03-07T03:21:22.478586+0000 mon.a (mon.0) 847 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:23.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[79878]: audit 2024-03-07T03:21:22.482098+0000 mon.a (mon.0) 848 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:21:23.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[79878]: audit 2024-03-07T03:21:22.483256+0000 mon.a (mon.0) 849 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:23.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[79878]: cephadm 2024-03-07T03:21:22.489704+0000 mgr.y (mgr.34107) 273 : cephadm 1 Deploying daemon osd.2 on smithi145 2024-03-07T03:21:23.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[79878]: cluster 2024-03-07T03:21:22.677007+0000 mgr.y (mgr.34107) 274 : cluster 0 pgmap v530: 297 pgs: 297 active+clean; 584 KiB data, 470 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:21:23.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[82329]: cluster 2024-03-07T03:21:22.362468+0000 mon.a (mon.0) 846 : cluster 0 osdmap e619: 8 total, 8 up, 8 in 2024-03-07T03:21:23.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[82329]: cephadm 2024-03-07T03:21:22.472313+0000 mgr.y (mgr.34107) 272 : cephadm 1 Upgrade: Updating osd.2 2024-03-07T03:21:23.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[82329]: audit 2024-03-07T03:21:22.478586+0000 mon.a (mon.0) 847 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:23.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[82329]: audit 2024-03-07T03:21:22.482098+0000 mon.a (mon.0) 848 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:21:23.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[82329]: audit 2024-03-07T03:21:22.483256+0000 mon.a (mon.0) 849 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:23.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[82329]: cephadm 2024-03-07T03:21:22.489704+0000 mgr.y (mgr.34107) 273 : cephadm 1 Deploying daemon osd.2 on smithi145 2024-03-07T03:21:23.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:23 smithi145 bash[82329]: cluster 2024-03-07T03:21:22.677007+0000 mgr.y (mgr.34107) 274 : cluster 0 pgmap v530: 297 pgs: 297 active+clean; 584 KiB data, 470 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:21:23.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:23 smithi189 bash[66969]: cluster 2024-03-07T03:21:22.362468+0000 mon.a (mon.0) 846 : cluster 0 osdmap e619: 8 total, 8 up, 8 in 2024-03-07T03:21:23.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:23 smithi189 bash[66969]: cephadm 2024-03-07T03:21:22.472313+0000 mgr.y (mgr.34107) 272 : cephadm 1 Upgrade: Updating osd.2 2024-03-07T03:21:23.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:23 smithi189 bash[66969]: audit 2024-03-07T03:21:22.478586+0000 mon.a (mon.0) 847 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:23.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:23 smithi189 bash[66969]: audit 2024-03-07T03:21:22.482098+0000 mon.a (mon.0) 848 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:21:23.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:23 smithi189 bash[66969]: audit 2024-03-07T03:21:22.483256+0000 mon.a (mon.0) 849 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:23.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:23 smithi189 bash[66969]: cephadm 2024-03-07T03:21:22.489704+0000 mgr.y (mgr.34107) 273 : cephadm 1 Deploying daemon osd.2 on smithi145 2024-03-07T03:21:23.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:23 smithi189 bash[66969]: cluster 2024-03-07T03:21:22.677007+0000 mgr.y (mgr.34107) 274 : cluster 0 pgmap v530: 297 pgs: 297 active+clean; 584 KiB data, 470 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:21:24.429 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 2 2024-03-07T03:21:24.429 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 2 2024-03-07T03:21:24.429 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap1 2024-03-07T03:21:24.429 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap2 2024-03-07T03:21:24.429 INFO:tasks.workunit.client.0.smithi145.stdout:found snap1 with size 2097152 2024-03-07T03:21:24.429 INFO:tasks.workunit.client.0.smithi145.stdout:found snap2 with size 4194304 2024-03-07T03:21:24.440 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 2 2024-03-07T03:21:24.440 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 2 2024-03-07T03:21:24.440 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap1-rename 2024-03-07T03:21:24.440 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap2 2024-03-07T03:21:24.440 INFO:tasks.workunit.client.0.smithi145.stdout:found snap1-rename with size 2097152 2024-03-07T03:21:24.440 INFO:tasks.workunit.client.0.smithi145.stdout:found snap2 with size 4194304 2024-03-07T03:21:24.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:24 smithi145 bash[79878]: cluster 2024-03-07T03:21:23.370770+0000 mon.a (mon.0) 850 : cluster 0 osdmap e620: 8 total, 8 up, 8 in 2024-03-07T03:21:24.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:24 smithi145 bash[79878]: audit 2024-03-07T03:21:23.585963+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.145:0/2374853475' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:24.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:24 smithi145 bash[82329]: cluster 2024-03-07T03:21:23.370770+0000 mon.a (mon.0) 850 : cluster 0 osdmap e620: 8 total, 8 up, 8 in 2024-03-07T03:21:24.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:24 smithi145 bash[82329]: audit 2024-03-07T03:21:23.585963+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.145:0/2374853475' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:24.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:24 smithi189 bash[66969]: cluster 2024-03-07T03:21:23.370770+0000 mon.a (mon.0) 850 : cluster 0 osdmap e620: 8 total, 8 up, 8 in 2024-03-07T03:21:24.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:24 smithi189 bash[66969]: audit 2024-03-07T03:21:23.585963+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.145:0/2374853475' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:25.435 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 1 2024-03-07T03:21:25.435 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:21:25.435 INFO:tasks.workunit.client.0.smithi145.stdout:snap: snap2-rename 2024-03-07T03:21:25.435 INFO:tasks.workunit.client.0.smithi145.stdout:found snap2-rename with size 4194304 2024-03-07T03:21:25.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:25 smithi145 bash[82329]: cluster 2024-03-07T03:21:24.386738+0000 mon.a (mon.0) 851 : cluster 0 osdmap e621: 8 total, 8 up, 8 in 2024-03-07T03:21:25.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:25 smithi145 bash[82329]: cluster 2024-03-07T03:21:24.677890+0000 mgr.y (mgr.34107) 275 : cluster 0 pgmap v533: 297 pgs: 297 active+clean; 584 KiB data, 471 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 6.0 KiB/s wr, 75 op/s 2024-03-07T03:21:25.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:25 smithi145 bash[79878]: cluster 2024-03-07T03:21:24.386738+0000 mon.a (mon.0) 851 : cluster 0 osdmap e621: 8 total, 8 up, 8 in 2024-03-07T03:21:25.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:25 smithi145 bash[79878]: cluster 2024-03-07T03:21:24.677890+0000 mgr.y (mgr.34107) 275 : cluster 0 pgmap v533: 297 pgs: 297 active+clean; 584 KiB data, 471 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 6.0 KiB/s wr, 75 op/s 2024-03-07T03:21:25.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:25 smithi189 bash[66969]: cluster 2024-03-07T03:21:24.386738+0000 mon.a (mon.0) 851 : cluster 0 osdmap e621: 8 total, 8 up, 8 in 2024-03-07T03:21:25.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:25 smithi189 bash[66969]: cluster 2024-03-07T03:21:24.677890+0000 mgr.y (mgr.34107) 275 : cluster 0 pgmap v533: 297 pgs: 297 active+clean; 584 KiB data, 471 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 6.0 KiB/s wr, 75 op/s 2024-03-07T03:21:26.406 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 0 2024-03-07T03:21:26.406 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 0 2024-03-07T03:21:26.426 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCreateLsRenameSnapPP (4033 ms) 2024-03-07T03:21:26.426 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ConcurrentCreatesUnvalidatedPool 2024-03-07T03:21:26.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:26 smithi145 bash[82329]: cluster 2024-03-07T03:21:25.398491+0000 mon.a (mon.0) 852 : cluster 0 osdmap e622: 8 total, 8 up, 8 in 2024-03-07T03:21:26.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:26 smithi145 bash[82329]: audit 2024-03-07T03:21:25.938449+0000 mon.a (mon.0) 853 : audit 0 from='client.? 172.21.15.145:0/1246116929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:26.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:26 smithi145 bash[79878]: cluster 2024-03-07T03:21:25.398491+0000 mon.a (mon.0) 852 : cluster 0 osdmap e622: 8 total, 8 up, 8 in 2024-03-07T03:21:26.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:26 smithi145 bash[79878]: audit 2024-03-07T03:21:25.938449+0000 mon.a (mon.0) 853 : audit 0 from='client.? 172.21.15.145:0/1246116929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:26.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:26 smithi189 bash[66969]: cluster 2024-03-07T03:21:25.398491+0000 mon.a (mon.0) 852 : cluster 0 osdmap e622: 8 total, 8 up, 8 in 2024-03-07T03:21:26.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:26 smithi189 bash[66969]: audit 2024-03-07T03:21:25.938449+0000 mon.a (mon.0) 853 : audit 0 from='client.? 172.21.15.145:0/1246116929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:27.560 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:21:27 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:27.560 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:21:27 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:27.560 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:27 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:27.561 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:27 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:27.561 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:21:27 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:27.561 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:21:27 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:27.562 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:21:27 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:27.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:27 smithi145 bash[79878]: cluster 2024-03-07T03:21:26.401792+0000 mon.a (mon.0) 854 : cluster 0 osdmap e623: 8 total, 8 up, 8 in 2024-03-07T03:21:27.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:27 smithi145 bash[79878]: cluster 2024-03-07T03:21:26.679062+0000 mgr.y (mgr.34107) 276 : cluster 0 pgmap v536: 297 pgs: 297 active+clean; 584 KiB data, 471 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 76 op/s 2024-03-07T03:21:27.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:27 smithi145 bash[79878]: cluster 2024-03-07T03:21:27.366500+0000 mon.a (mon.0) 855 : cluster 0 osdmap e624: 8 total, 8 up, 8 in 2024-03-07T03:21:27.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:27 smithi145 bash[79878]: audit 2024-03-07T03:21:27.368394+0000 mon.c (mon.2) 20 : audit 1 from='client.? 172.21.15.145:0/1060279695' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:27.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:27 smithi145 bash[79878]: audit 2024-03-07T03:21:27.369113+0000 mon.a (mon.0) 856 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:27.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:27 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:27.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:27 smithi145 bash[82329]: cluster 2024-03-07T03:21:26.401792+0000 mon.a (mon.0) 854 : cluster 0 osdmap e623: 8 total, 8 up, 8 in 2024-03-07T03:21:27.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:27 smithi145 bash[82329]: cluster 2024-03-07T03:21:26.679062+0000 mgr.y (mgr.34107) 276 : cluster 0 pgmap v536: 297 pgs: 297 active+clean; 584 KiB data, 471 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 76 op/s 2024-03-07T03:21:27.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:27 smithi145 bash[82329]: cluster 2024-03-07T03:21:27.366500+0000 mon.a (mon.0) 855 : cluster 0 osdmap e624: 8 total, 8 up, 8 in 2024-03-07T03:21:27.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:27 smithi145 bash[82329]: audit 2024-03-07T03:21:27.368394+0000 mon.c (mon.2) 20 : audit 1 from='client.? 172.21.15.145:0/1060279695' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:27.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:27 smithi145 bash[82329]: audit 2024-03-07T03:21:27.369113+0000 mon.a (mon.0) 856 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:27.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:27 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:27.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:27 smithi189 bash[66969]: cluster 2024-03-07T03:21:26.401792+0000 mon.a (mon.0) 854 : cluster 0 osdmap e623: 8 total, 8 up, 8 in 2024-03-07T03:21:27.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:27 smithi189 bash[66969]: cluster 2024-03-07T03:21:26.679062+0000 mgr.y (mgr.34107) 276 : cluster 0 pgmap v536: 297 pgs: 297 active+clean; 584 KiB data, 471 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 6.2 KiB/s wr, 76 op/s 2024-03-07T03:21:27.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:27 smithi189 bash[66969]: cluster 2024-03-07T03:21:27.366500+0000 mon.a (mon.0) 855 : cluster 0 osdmap e624: 8 total, 8 up, 8 in 2024-03-07T03:21:27.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:27 smithi189 bash[66969]: audit 2024-03-07T03:21:27.368394+0000 mon.c (mon.2) 20 : audit 1 from='client.? 172.21.15.145:0/1060279695' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:27.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:27 smithi189 bash[66969]: audit 2024-03-07T03:21:27.369113+0000 mon.a (mon.0) 856 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:28.060 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:27 smithi145 systemd[1]: Stopping Ceph osd.2 for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:21:28.060 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:27 smithi145 bash[45104]: debug 2024-03-07T03:21:27.682+0000 7f9fcb311700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:21:28.060 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:27 smithi145 bash[45104]: debug 2024-03-07T03:21:27.682+0000 7f9fcb311700 -1 osd.2 624 *** Got signal Terminated *** 2024-03-07T03:21:28.060 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:27 smithi145 bash[45104]: debug 2024-03-07T03:21:27.682+0000 7f9fcb311700 -1 osd.2 624 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:21:28.363 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:21:28.363 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:21:28.363 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:21:28.363 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:21:28.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[82329]: cluster 2024-03-07T03:21:27.688615+0000 mon.a (mon.0) 857 : cluster 1 osd.2 marked itself down and dead 2024-03-07T03:21:28.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[82329]: audit 2024-03-07T03:21:28.298298+0000 mon.a (mon.0) 858 : audit 0 from='client.? 172.21.15.145:0/3861573592' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:28.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[82329]: cluster 2024-03-07T03:21:28.359723+0000 mon.a (mon.0) 859 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:21:28.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[82329]: cluster 2024-03-07T03:21:28.359767+0000 mon.a (mon.0) 860 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:28.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[82329]: audit 2024-03-07T03:21:28.361896+0000 mon.a (mon.0) 861 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:28.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[82329]: cluster 2024-03-07T03:21:28.367019+0000 mon.a (mon.0) 862 : cluster 0 osdmap e625: 8 total, 7 up, 8 in 2024-03-07T03:21:28.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[82329]: cluster 2024-03-07T03:21:28.375862+0000 mon.a (mon.0) 863 : cluster 1 osd.2 marked itself dead as of e625 2024-03-07T03:21:28.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[79878]: debug 2024-03-07T03:21:28.354+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:28.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[79878]: cluster 2024-03-07T03:21:27.688615+0000 mon.a (mon.0) 857 : cluster 1 osd.2 marked itself down and dead 2024-03-07T03:21:28.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[79878]: audit 2024-03-07T03:21:28.298298+0000 mon.a (mon.0) 858 : audit 0 from='client.? 172.21.15.145:0/3861573592' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:28.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[79878]: cluster 2024-03-07T03:21:28.359723+0000 mon.a (mon.0) 859 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:21:28.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[79878]: cluster 2024-03-07T03:21:28.359767+0000 mon.a (mon.0) 860 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:28.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[79878]: audit 2024-03-07T03:21:28.361896+0000 mon.a (mon.0) 861 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:28.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[79878]: cluster 2024-03-07T03:21:28.367019+0000 mon.a (mon.0) 862 : cluster 0 osdmap e625: 8 total, 7 up, 8 in 2024-03-07T03:21:28.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:28 smithi145 bash[79878]: cluster 2024-03-07T03:21:28.375862+0000 mon.a (mon.0) 863 : cluster 1 osd.2 marked itself dead as of e625 2024-03-07T03:21:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:28 smithi189 bash[66969]: cluster 2024-03-07T03:21:27.688615+0000 mon.a (mon.0) 857 : cluster 1 osd.2 marked itself down and dead 2024-03-07T03:21:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:28 smithi189 bash[66969]: audit 2024-03-07T03:21:28.298298+0000 mon.a (mon.0) 858 : audit 0 from='client.? 172.21.15.145:0/3861573592' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:28 smithi189 bash[66969]: cluster 2024-03-07T03:21:28.359723+0000 mon.a (mon.0) 859 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:21:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:28 smithi189 bash[66969]: cluster 2024-03-07T03:21:28.359767+0000 mon.a (mon.0) 860 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:28.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:28 smithi189 bash[66969]: audit 2024-03-07T03:21:28.361896+0000 mon.a (mon.0) 861 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:28.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:28 smithi189 bash[66969]: cluster 2024-03-07T03:21:28.367019+0000 mon.a (mon.0) 862 : cluster 0 osdmap e625: 8 total, 7 up, 8 in 2024-03-07T03:21:28.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:28 smithi189 bash[66969]: cluster 2024-03-07T03:21:28.375862+0000 mon.a (mon.0) 863 : cluster 1 osd.2 marked itself dead as of e625 2024-03-07T03:21:29.703 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:29 smithi145 bash[79878]: cluster 2024-03-07T03:21:28.682327+0000 mgr.y (mgr.34107) 277 : cluster 0 pgmap v539: 329 pgs: 70 peering, 10 stale+active+clean, 11 creating+peering, 10 unknown, 228 active+clean; 584 KiB data, 471 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-07T03:21:29.703 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:29 smithi145 bash[79878]: cluster 2024-03-07T03:21:29.372061+0000 mon.a (mon.0) 864 : cluster 0 osdmap e626: 8 total, 7 up, 8 in 2024-03-07T03:21:29.703 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:29 smithi145 bash[82329]: cluster 2024-03-07T03:21:28.682327+0000 mgr.y (mgr.34107) 277 : cluster 0 pgmap v539: 329 pgs: 70 peering, 10 stale+active+clean, 11 creating+peering, 10 unknown, 228 active+clean; 584 KiB data, 471 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-07T03:21:29.703 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:29 smithi145 bash[82329]: cluster 2024-03-07T03:21:29.372061+0000 mon.a (mon.0) 864 : cluster 0 osdmap e626: 8 total, 7 up, 8 in 2024-03-07T03:21:29.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:29 smithi189 bash[66969]: cluster 2024-03-07T03:21:28.682327+0000 mgr.y (mgr.34107) 277 : cluster 0 pgmap v539: 329 pgs: 70 peering, 10 stale+active+clean, 11 creating+peering, 10 unknown, 228 active+clean; 584 KiB data, 471 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-07T03:21:29.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:29 smithi189 bash[66969]: cluster 2024-03-07T03:21:29.372061+0000 mon.a (mon.0) 864 : cluster 0 osdmap e626: 8 total, 7 up, 8 in 2024-03-07T03:21:29.985 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:29 smithi145 bash[98965]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-2 2024-03-07T03:21:29.986 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:29 smithi145 bash[99057]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-2 2024-03-07T03:21:30.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:30 smithi145 bash[82329]: cluster 2024-03-07T03:21:29.419129+0000 mon.a (mon.0) 865 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:21:30.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:30 smithi145 bash[82329]: cluster 2024-03-07T03:21:30.375056+0000 mon.a (mon.0) 866 : cluster 0 osdmap e627: 8 total, 7 up, 8 in 2024-03-07T03:21:30.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:30 smithi145 bash[79878]: cluster 2024-03-07T03:21:29.419129+0000 mon.a (mon.0) 865 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:21:30.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:30 smithi145 bash[79878]: cluster 2024-03-07T03:21:30.375056+0000 mon.a (mon.0) 866 : cluster 0 osdmap e627: 8 total, 7 up, 8 in 2024-03-07T03:21:30.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:30 smithi189 bash[66969]: cluster 2024-03-07T03:21:29.419129+0000 mon.a (mon.0) 865 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:21:30.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:30 smithi189 bash[66969]: cluster 2024-03-07T03:21:30.375056+0000 mon.a (mon.0) 866 : cluster 0 osdmap e627: 8 total, 7 up, 8 in 2024-03-07T03:21:31.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:31 smithi145 bash[79878]: audit 2024-03-07T03:21:30.665970+0000 mon.a (mon.0) 867 : audit 0 from='client.? 172.21.15.145:0/1747358152' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:31.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:31 smithi145 bash[79878]: cluster 2024-03-07T03:21:30.683062+0000 mgr.y (mgr.34107) 278 : cluster 0 pgmap v542: 329 pgs: 70 peering, 10 stale+active+clean, 11 creating+peering, 10 unknown, 228 active+clean; 584 KiB data, 471 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-07T03:21:31.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:31 smithi145 bash[79878]: cluster 2024-03-07T03:21:31.375318+0000 mon.a (mon.0) 868 : cluster 0 osdmap e628: 8 total, 7 up, 8 in 2024-03-07T03:21:31.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:31 smithi145 bash[82329]: audit 2024-03-07T03:21:30.665970+0000 mon.a (mon.0) 867 : audit 0 from='client.? 172.21.15.145:0/1747358152' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:31.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:31 smithi145 bash[82329]: cluster 2024-03-07T03:21:30.683062+0000 mgr.y (mgr.34107) 278 : cluster 0 pgmap v542: 329 pgs: 70 peering, 10 stale+active+clean, 11 creating+peering, 10 unknown, 228 active+clean; 584 KiB data, 471 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-07T03:21:31.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:31 smithi145 bash[82329]: cluster 2024-03-07T03:21:31.375318+0000 mon.a (mon.0) 868 : cluster 0 osdmap e628: 8 total, 7 up, 8 in 2024-03-07T03:21:31.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:31 smithi189 bash[66969]: audit 2024-03-07T03:21:30.665970+0000 mon.a (mon.0) 867 : audit 0 from='client.? 172.21.15.145:0/1747358152' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:31.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:31 smithi189 bash[66969]: cluster 2024-03-07T03:21:30.683062+0000 mgr.y (mgr.34107) 278 : cluster 0 pgmap v542: 329 pgs: 70 peering, 10 stale+active+clean, 11 creating+peering, 10 unknown, 228 active+clean; 584 KiB data, 471 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-07T03:21:31.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:31 smithi189 bash[66969]: cluster 2024-03-07T03:21:31.375318+0000 mon.a (mon.0) 868 : cluster 0 osdmap e628: 8 total, 7 up, 8 in 2024-03-07T03:21:31.891 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ConcurrentCreatesUnvalidatedPool (5465 ms) 2024-03-07T03:21:31.891 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.RemoveFullTry 2024-03-07T03:21:32.512 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:32 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:32.512 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:32 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:32.513 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:21:32 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:32.513 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:21:32 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:32.513 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:21:32 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:32.514 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:21:32 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:32.514 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:21:32 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:32.514 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:32 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:32.514 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:32 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.2.service: Deactivated successfully. 2024-03-07T03:21:32.514 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:32 smithi145 systemd[1]: Stopped Ceph osd.2 for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:21:32.515 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:32 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:21:32 smithi189 bash[59814]: ts=2024-03-07T03:21:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:21:32.810 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:32 smithi145 systemd[1]: Started Ceph osd.2 for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:21:32.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:21:32] "GET /metrics HTTP/1.1" 200 55701 "" "Prometheus/2.43.0" 2024-03-07T03:21:33.413 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:21:33.632 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[79878]: cluster 2024-03-07T03:21:32.370407+0000 mon.a (mon.0) 869 : cluster 0 osdmap e629: 8 total, 7 up, 8 in 2024-03-07T03:21:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[79878]: audit 2024-03-07T03:21:32.371853+0000 mon.c (mon.2) 21 : audit 1 from='client.? 172.21.15.145:0/3774034288' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[79878]: audit 2024-03-07T03:21:32.372290+0000 mon.a (mon.0) 870 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[79878]: audit 2024-03-07T03:21:32.571635+0000 mon.a (mon.0) 871 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[79878]: audit 2024-03-07T03:21:32.577460+0000 mon.a (mon.0) 872 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[79878]: audit 2024-03-07T03:21:32.581133+0000 mon.a (mon.0) 873 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[79878]: audit 2024-03-07T03:21:32.643610+0000 mon.a (mon.0) 874 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[79878]: audit 2024-03-07T03:21:32.644822+0000 mon.a (mon.0) 875 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[79878]: cluster 2024-03-07T03:21:32.683852+0000 mgr.y (mgr.34107) 279 : cluster 0 pgmap v545: 361 pgs: 13 active+undersized+degraded, 26 active+undersized, 70 peering, 32 unknown, 11 creating+peering, 209 active+clean; 584 KiB data, 472 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 5.0 KiB/s wr, 48 op/s; 19/483 objects degraded (3.934%) 2024-03-07T03:21:33.633 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[79878]: audit 2024-03-07T03:21:33.011560+0000 mon.a (mon.0) 876 : audit 0 from='client.? 172.21.15.145:0/1836186757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[82329]: cluster 2024-03-07T03:21:32.370407+0000 mon.a (mon.0) 869 : cluster 0 osdmap e629: 8 total, 7 up, 8 in 2024-03-07T03:21:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[82329]: audit 2024-03-07T03:21:32.371853+0000 mon.c (mon.2) 21 : audit 1 from='client.? 172.21.15.145:0/3774034288' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[82329]: audit 2024-03-07T03:21:32.372290+0000 mon.a (mon.0) 870 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[82329]: audit 2024-03-07T03:21:32.571635+0000 mon.a (mon.0) 871 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[82329]: audit 2024-03-07T03:21:32.577460+0000 mon.a (mon.0) 872 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[82329]: audit 2024-03-07T03:21:32.581133+0000 mon.a (mon.0) 873 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[82329]: audit 2024-03-07T03:21:32.643610+0000 mon.a (mon.0) 874 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[82329]: audit 2024-03-07T03:21:32.644822+0000 mon.a (mon.0) 875 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[82329]: cluster 2024-03-07T03:21:32.683852+0000 mgr.y (mgr.34107) 279 : cluster 0 pgmap v545: 361 pgs: 13 active+undersized+degraded, 26 active+undersized, 70 peering, 32 unknown, 11 creating+peering, 209 active+clean; 584 KiB data, 472 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 5.0 KiB/s wr, 48 op/s; 19/483 objects degraded (3.934%) 2024-03-07T03:21:33.634 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[82329]: audit 2024-03-07T03:21:33.011560+0000 mon.a (mon.0) 876 : audit 0 from='client.? 172.21.15.145:0/1836186757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:33.634 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[72238]: [07/Mar/2024:03:21:33] ENGINE Bus STOPPING 2024-03-07T03:21:33.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[72238]: [07/Mar/2024:03:21:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:21:33.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[72238]: [07/Mar/2024:03:21:33] ENGINE Bus STOPPED 2024-03-07T03:21:33.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[72238]: [07/Mar/2024:03:21:33] ENGINE Bus STARTING 2024-03-07T03:21:33.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[72238]: [07/Mar/2024:03:21:33] ENGINE Serving on http://:::9283 2024-03-07T03:21:33.635 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:33 smithi145 bash[72238]: [07/Mar/2024:03:21:33] ENGINE Bus STARTED 2024-03-07T03:21:33.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:33 smithi189 bash[66969]: cluster 2024-03-07T03:21:32.370407+0000 mon.a (mon.0) 869 : cluster 0 osdmap e629: 8 total, 7 up, 8 in 2024-03-07T03:21:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:33 smithi189 bash[66969]: audit 2024-03-07T03:21:32.371853+0000 mon.c (mon.2) 21 : audit 1 from='client.? 172.21.15.145:0/3774034288' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:33 smithi189 bash[66969]: audit 2024-03-07T03:21:32.372290+0000 mon.a (mon.0) 870 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:33 smithi189 bash[66969]: audit 2024-03-07T03:21:32.571635+0000 mon.a (mon.0) 871 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:33 smithi189 bash[66969]: audit 2024-03-07T03:21:32.577460+0000 mon.a (mon.0) 872 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:33 smithi189 bash[66969]: audit 2024-03-07T03:21:32.581133+0000 mon.a (mon.0) 873 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:33 smithi189 bash[66969]: audit 2024-03-07T03:21:32.643610+0000 mon.a (mon.0) 874 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:33 smithi189 bash[66969]: audit 2024-03-07T03:21:32.644822+0000 mon.a (mon.0) 875 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:33 smithi189 bash[66969]: cluster 2024-03-07T03:21:32.683852+0000 mgr.y (mgr.34107) 279 : cluster 0 pgmap v545: 361 pgs: 13 active+undersized+degraded, 26 active+undersized, 70 peering, 32 unknown, 11 creating+peering, 209 active+clean; 584 KiB data, 472 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 5.0 KiB/s wr, 48 op/s; 19/483 objects degraded (3.934%) 2024-03-07T03:21:33.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:33 smithi189 bash[66969]: audit 2024-03-07T03:21:33.011560+0000 mon.a (mon.0) 876 : audit 0 from='client.? 172.21.15.145:0/1836186757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:34.489 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[79878]: audit 2024-03-07T03:21:33.366640+0000 mon.a (mon.0) 877 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:34.490 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[79878]: cluster 2024-03-07T03:21:33.373653+0000 mon.a (mon.0) 878 : cluster 0 osdmap e630: 8 total, 7 up, 8 in 2024-03-07T03:21:34.490 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[79878]: audit 2024-03-07T03:21:33.424653+0000 mon.a (mon.0) 879 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' 2024-03-07T03:21:34.490 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[79878]: cluster 2024-03-07T03:21:33.641674+0000 mon.a (mon.0) 880 : cluster 3 Health check failed: Degraded data redundancy: 19/483 objects degraded (3.934%), 13 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:34.490 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[79878]: cluster 2024-03-07T03:21:33.641712+0000 mon.a (mon.0) 881 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:21:34.490 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[82329]: audit 2024-03-07T03:21:33.366640+0000 mon.a (mon.0) 877 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:34.490 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[82329]: cluster 2024-03-07T03:21:33.373653+0000 mon.a (mon.0) 878 : cluster 0 osdmap e630: 8 total, 7 up, 8 in 2024-03-07T03:21:34.490 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[82329]: audit 2024-03-07T03:21:33.424653+0000 mon.a (mon.0) 879 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' 2024-03-07T03:21:34.490 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[82329]: cluster 2024-03-07T03:21:33.641674+0000 mon.a (mon.0) 880 : cluster 3 Health check failed: Degraded data redundancy: 19/483 objects degraded (3.934%), 13 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:34.490 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[82329]: cluster 2024-03-07T03:21:33.641712+0000 mon.a (mon.0) 881 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:21:34.491 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[99253]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-07T03:21:34.491 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[99253]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-07T03:21:34.491 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[99253]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-07T03:21:34.491 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[99253]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-07T03:21:34.491 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[99253]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-07T03:21:34.491 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[99253]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-07T03:21:34.491 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:34 smithi145 bash[99253]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-07T03:21:34.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:34 smithi189 bash[66969]: audit 2024-03-07T03:21:33.366640+0000 mon.a (mon.0) 877 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:34.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:34 smithi189 bash[66969]: cluster 2024-03-07T03:21:33.373653+0000 mon.a (mon.0) 878 : cluster 0 osdmap e630: 8 total, 7 up, 8 in 2024-03-07T03:21:34.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:34 smithi189 bash[66969]: audit 2024-03-07T03:21:33.424653+0000 mon.a (mon.0) 879 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' 2024-03-07T03:21:34.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:34 smithi189 bash[66969]: cluster 2024-03-07T03:21:33.641674+0000 mon.a (mon.0) 880 : cluster 3 Health check failed: Degraded data redundancy: 19/483 objects degraded (3.934%), 13 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:34.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:34 smithi189 bash[66969]: cluster 2024-03-07T03:21:33.641712+0000 mon.a (mon.0) 881 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:21:35.573 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:21:35.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:35 smithi145 bash[82329]: cluster 2024-03-07T03:21:34.409831+0000 mon.a (mon.0) 882 : cluster 0 osdmap e631: 8 total, 7 up, 8 in 2024-03-07T03:21:35.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:35 smithi145 bash[82329]: cluster 2024-03-07T03:21:34.685214+0000 mgr.y (mgr.34107) 280 : cluster 0 pgmap v548: 361 pgs: 42 active+undersized+degraded, 87 active+undersized, 9 unknown, 223 active+clean; 584 KiB data, 472 MiB used, 715 GiB / 715 GiB avail; 110 KiB/s rd, 11 KiB/s wr, 157 op/s; 58/489 objects degraded (11.861%) 2024-03-07T03:21:35.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:35 smithi145 bash[82329]: audit 2024-03-07T03:21:35.366235+0000 mon.a (mon.0) 883 : audit 0 from='client.? 172.21.15.145:0/2379444264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:35.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:35 smithi145 bash[79878]: cluster 2024-03-07T03:21:34.409831+0000 mon.a (mon.0) 882 : cluster 0 osdmap e631: 8 total, 7 up, 8 in 2024-03-07T03:21:35.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:35 smithi145 bash[79878]: cluster 2024-03-07T03:21:34.685214+0000 mgr.y (mgr.34107) 280 : cluster 0 pgmap v548: 361 pgs: 42 active+undersized+degraded, 87 active+undersized, 9 unknown, 223 active+clean; 584 KiB data, 472 MiB used, 715 GiB / 715 GiB avail; 110 KiB/s rd, 11 KiB/s wr, 157 op/s; 58/489 objects degraded (11.861%) 2024-03-07T03:21:35.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:35 smithi145 bash[79878]: audit 2024-03-07T03:21:35.366235+0000 mon.a (mon.0) 883 : audit 0 from='client.? 172.21.15.145:0/2379444264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:35.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:35 smithi189 bash[66969]: cluster 2024-03-07T03:21:34.409831+0000 mon.a (mon.0) 882 : cluster 0 osdmap e631: 8 total, 7 up, 8 in 2024-03-07T03:21:35.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:35 smithi189 bash[66969]: cluster 2024-03-07T03:21:34.685214+0000 mgr.y (mgr.34107) 280 : cluster 0 pgmap v548: 361 pgs: 42 active+undersized+degraded, 87 active+undersized, 9 unknown, 223 active+clean; 584 KiB data, 472 MiB used, 715 GiB / 715 GiB avail; 110 KiB/s rd, 11 KiB/s wr, 157 op/s; 58/489 objects degraded (11.861%) 2024-03-07T03:21:35.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:35 smithi189 bash[66969]: audit 2024-03-07T03:21:35.366235+0000 mon.a (mon.0) 883 : audit 0 from='client.? 172.21.15.145:0/2379444264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:36.310 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:35 smithi145 bash[99521]: debug 2024-03-07T03:21:35.918+0000 7f935fa63700 1 -- 172.21.15.145:0/2778401621 <== mon.0 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559a32a5d680 con 0x559a32a99000 2024-03-07T03:21:36.810 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:36 smithi145 bash[99521]: debug 2024-03-07T03:21:36.486+0000 7f9368bfb7c0 -1 Falling back to public interface 2024-03-07T03:21:36.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:36 smithi145 bash[82329]: cluster 2024-03-07T03:21:35.404201+0000 mon.a (mon.0) 884 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-07T03:21:36.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:36 smithi145 bash[82329]: cluster 2024-03-07T03:21:35.404238+0000 mon.a (mon.0) 885 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:21:36.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:36 smithi145 bash[82329]: cluster 2024-03-07T03:21:35.424716+0000 mon.a (mon.0) 886 : cluster 0 osdmap e632: 8 total, 7 up, 8 in 2024-03-07T03:21:36.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:36 smithi145 bash[82329]: audit 2024-03-07T03:21:35.598151+0000 mon.a (mon.0) 887 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi145-68337-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:21:36.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:36 smithi145 bash[79878]: cluster 2024-03-07T03:21:35.404201+0000 mon.a (mon.0) 884 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-07T03:21:36.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:36 smithi145 bash[79878]: cluster 2024-03-07T03:21:35.404238+0000 mon.a (mon.0) 885 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:21:36.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:36 smithi145 bash[79878]: cluster 2024-03-07T03:21:35.424716+0000 mon.a (mon.0) 886 : cluster 0 osdmap e632: 8 total, 7 up, 8 in 2024-03-07T03:21:36.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:36 smithi145 bash[79878]: audit 2024-03-07T03:21:35.598151+0000 mon.a (mon.0) 887 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi145-68337-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:21:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:36 smithi189 bash[66969]: cluster 2024-03-07T03:21:35.404201+0000 mon.a (mon.0) 884 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-07T03:21:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:36 smithi189 bash[66969]: cluster 2024-03-07T03:21:35.404238+0000 mon.a (mon.0) 885 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:21:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:36 smithi189 bash[66969]: cluster 2024-03-07T03:21:35.424716+0000 mon.a (mon.0) 886 : cluster 0 osdmap e632: 8 total, 7 up, 8 in 2024-03-07T03:21:36.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:36 smithi189 bash[66969]: audit 2024-03-07T03:21:35.598151+0000 mon.a (mon.0) 887 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi145-68337-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:21:37.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:37 smithi145 bash[79878]: audit 2024-03-07T03:21:36.417545+0000 mon.a (mon.0) 888 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi145-68337-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-07T03:21:37.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:37 smithi145 bash[79878]: cluster 2024-03-07T03:21:36.424278+0000 mon.a (mon.0) 889 : cluster 0 osdmap e633: 8 total, 7 up, 8 in 2024-03-07T03:21:37.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:37 smithi145 bash[79878]: cluster 2024-03-07T03:21:36.687032+0000 mgr.y (mgr.34107) 281 : cluster 0 pgmap v551: 361 pgs: 41 active+undersized+degraded, 90 active+undersized, 6 unknown, 224 active+clean; 584 KiB data, 472 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 6.5 KiB/s wr, 135 op/s; 57/486 objects degraded (11.728%) 2024-03-07T03:21:37.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:37 smithi145 bash[79878]: cluster 2024-03-07T03:21:37.373603+0000 mon.a (mon.0) 890 : cluster 0 osdmap e634: 8 total, 7 up, 8 in 2024-03-07T03:21:37.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:37 smithi145 bash[82329]: audit 2024-03-07T03:21:36.417545+0000 mon.a (mon.0) 888 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi145-68337-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-07T03:21:37.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:37 smithi145 bash[82329]: cluster 2024-03-07T03:21:36.424278+0000 mon.a (mon.0) 889 : cluster 0 osdmap e633: 8 total, 7 up, 8 in 2024-03-07T03:21:37.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:37 smithi145 bash[82329]: cluster 2024-03-07T03:21:36.687032+0000 mgr.y (mgr.34107) 281 : cluster 0 pgmap v551: 361 pgs: 41 active+undersized+degraded, 90 active+undersized, 6 unknown, 224 active+clean; 584 KiB data, 472 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 6.5 KiB/s wr, 135 op/s; 57/486 objects degraded (11.728%) 2024-03-07T03:21:37.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:37 smithi145 bash[82329]: cluster 2024-03-07T03:21:37.373603+0000 mon.a (mon.0) 890 : cluster 0 osdmap e634: 8 total, 7 up, 8 in 2024-03-07T03:21:37.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:37 smithi189 bash[66969]: audit 2024-03-07T03:21:36.417545+0000 mon.a (mon.0) 888 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi145-68337-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-07T03:21:37.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:37 smithi189 bash[66969]: cluster 2024-03-07T03:21:36.424278+0000 mon.a (mon.0) 889 : cluster 0 osdmap e633: 8 total, 7 up, 8 in 2024-03-07T03:21:37.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:37 smithi189 bash[66969]: cluster 2024-03-07T03:21:36.687032+0000 mgr.y (mgr.34107) 281 : cluster 0 pgmap v551: 361 pgs: 41 active+undersized+degraded, 90 active+undersized, 6 unknown, 224 active+clean; 584 KiB data, 472 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s rd, 6.5 KiB/s wr, 135 op/s; 57/486 objects degraded (11.728%) 2024-03-07T03:21:37.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:37 smithi189 bash[66969]: cluster 2024-03-07T03:21:37.373603+0000 mon.a (mon.0) 890 : cluster 0 osdmap e634: 8 total, 7 up, 8 in 2024-03-07T03:21:38.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:38 smithi145 bash[82329]: audit 2024-03-07T03:21:37.735535+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.145:0/1064204393' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:38.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:38 smithi145 bash[82329]: audit 2024-03-07T03:21:38.366932+0000 mon.a (mon.0) 891 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:38.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:38 smithi145 bash[82329]: audit 2024-03-07T03:21:38.372271+0000 mon.a (mon.0) 892 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:38.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:38 smithi145 bash[79878]: audit 2024-03-07T03:21:37.735535+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.145:0/1064204393' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:38.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:38 smithi145 bash[79878]: audit 2024-03-07T03:21:38.366932+0000 mon.a (mon.0) 891 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:38.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:38 smithi145 bash[79878]: audit 2024-03-07T03:21:38.372271+0000 mon.a (mon.0) 892 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:38.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:38 smithi189 bash[66969]: audit 2024-03-07T03:21:37.735535+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.145:0/1064204393' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:38.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:38 smithi189 bash[66969]: audit 2024-03-07T03:21:38.366932+0000 mon.a (mon.0) 891 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:38.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:38 smithi189 bash[66969]: audit 2024-03-07T03:21:38.372271+0000 mon.a (mon.0) 892 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:40.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:39 smithi145 bash[79878]: audit 2024-03-07T03:21:38.622019+0000 mon.a (mon.0) 893 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:40.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:39 smithi145 bash[79878]: audit 2024-03-07T03:21:38.628021+0000 mon.a (mon.0) 894 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:40.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:39 smithi145 bash[79878]: cluster 2024-03-07T03:21:38.688845+0000 mgr.y (mgr.34107) 282 : cluster 0 pgmap v553: 361 pgs: 43 active+undersized+degraded, 91 active+undersized, 227 active+clean; 3.6 MiB data, 481 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 721 KiB/s wr, 72 op/s; 59/504 objects degraded (11.706%) 2024-03-07T03:21:40.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:39 smithi145 bash[79878]: audit 2024-03-07T03:21:39.477072+0000 mon.a (mon.0) 895 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:40.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:39 smithi145 bash[79878]: audit 2024-03-07T03:21:39.485126+0000 mon.a (mon.0) 896 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:40.060 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:39 smithi145 bash[99521]: debug 2024-03-07T03:21:39.698+0000 7f9368bfb7c0 -1 osd.2 625 log_to_monitors true 2024-03-07T03:21:40.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:39 smithi145 bash[82329]: audit 2024-03-07T03:21:38.622019+0000 mon.a (mon.0) 893 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:40.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:39 smithi145 bash[82329]: audit 2024-03-07T03:21:38.628021+0000 mon.a (mon.0) 894 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:40.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:39 smithi145 bash[82329]: cluster 2024-03-07T03:21:38.688845+0000 mgr.y (mgr.34107) 282 : cluster 0 pgmap v553: 361 pgs: 43 active+undersized+degraded, 91 active+undersized, 227 active+clean; 3.6 MiB data, 481 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 721 KiB/s wr, 72 op/s; 59/504 objects degraded (11.706%) 2024-03-07T03:21:40.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:39 smithi145 bash[82329]: audit 2024-03-07T03:21:39.477072+0000 mon.a (mon.0) 895 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:40.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:39 smithi145 bash[82329]: audit 2024-03-07T03:21:39.485126+0000 mon.a (mon.0) 896 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:39 smithi189 bash[66969]: audit 2024-03-07T03:21:38.622019+0000 mon.a (mon.0) 893 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:39 smithi189 bash[66969]: audit 2024-03-07T03:21:38.628021+0000 mon.a (mon.0) 894 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:39 smithi189 bash[66969]: cluster 2024-03-07T03:21:38.688845+0000 mgr.y (mgr.34107) 282 : cluster 0 pgmap v553: 361 pgs: 43 active+undersized+degraded, 91 active+undersized, 227 active+clean; 3.6 MiB data, 481 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 721 KiB/s wr, 72 op/s; 59/504 objects degraded (11.706%) 2024-03-07T03:21:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:39 smithi189 bash[66969]: audit 2024-03-07T03:21:39.477072+0000 mon.a (mon.0) 895 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:39 smithi189 bash[66969]: audit 2024-03-07T03:21:39.485126+0000 mon.a (mon.0) 896 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:41.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:40 smithi145 bash[79878]: cluster 2024-03-07T03:21:39.628123+0000 mon.a (mon.0) 897 : cluster 3 Health check update: Degraded data redundancy: 59/504 objects degraded (11.706%), 43 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:41.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:40 smithi145 bash[79878]: audit 2024-03-07T03:21:39.705457+0000 mon.c (mon.2) 23 : audit 1 from='osd.2 [v2:172.21.15.145:6818/3198195141,v1:172.21.15.145:6819/3198195141]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:21:41.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:40 smithi145 bash[79878]: audit 2024-03-07T03:21:39.705838+0000 mon.a (mon.0) 898 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:21:41.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:40 smithi145 bash[79878]: audit 2024-03-07T03:21:40.080922+0000 mon.a (mon.0) 899 : audit 0 from='client.? 172.21.15.145:0/4063995522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:41.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:40 smithi145 bash[82329]: cluster 2024-03-07T03:21:39.628123+0000 mon.a (mon.0) 897 : cluster 3 Health check update: Degraded data redundancy: 59/504 objects degraded (11.706%), 43 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:41.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:40 smithi145 bash[82329]: audit 2024-03-07T03:21:39.705457+0000 mon.c (mon.2) 23 : audit 1 from='osd.2 [v2:172.21.15.145:6818/3198195141,v1:172.21.15.145:6819/3198195141]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:21:41.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:40 smithi145 bash[82329]: audit 2024-03-07T03:21:39.705838+0000 mon.a (mon.0) 898 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:21:41.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:40 smithi145 bash[82329]: audit 2024-03-07T03:21:40.080922+0000 mon.a (mon.0) 899 : audit 0 from='client.? 172.21.15.145:0/4063995522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:40 smithi189 bash[66969]: cluster 2024-03-07T03:21:39.628123+0000 mon.a (mon.0) 897 : cluster 3 Health check update: Degraded data redundancy: 59/504 objects degraded (11.706%), 43 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:40 smithi189 bash[66969]: audit 2024-03-07T03:21:39.705457+0000 mon.c (mon.2) 23 : audit 1 from='osd.2 [v2:172.21.15.145:6818/3198195141,v1:172.21.15.145:6819/3198195141]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:21:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:40 smithi189 bash[66969]: audit 2024-03-07T03:21:39.705838+0000 mon.a (mon.0) 898 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:21:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:40 smithi189 bash[66969]: audit 2024-03-07T03:21:40.080922+0000 mon.a (mon.0) 899 : audit 0 from='client.? 172.21.15.145:0/4063995522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:41.560 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:41 smithi145 bash[99521]: debug 2024-03-07T03:21:41.210+0000 7f935604b700 -1 osd.2 625 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-07T03:21:42.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:41 smithi145 bash[79878]: audit 2024-03-07T03:21:40.636378+0000 mon.a (mon.0) 900 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-07T03:21:42.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:41 smithi145 bash[79878]: cluster 2024-03-07T03:21:40.645757+0000 mon.a (mon.0) 901 : cluster 0 osdmap e635: 8 total, 7 up, 8 in 2024-03-07T03:21:42.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:41 smithi145 bash[79878]: audit 2024-03-07T03:21:40.647212+0000 mon.c (mon.2) 24 : audit 1 from='osd.2 [v2:172.21.15.145:6818/3198195141,v1:172.21.15.145:6819/3198195141]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:21:42.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:41 smithi145 bash[79878]: audit 2024-03-07T03:21:40.649213+0000 mon.a (mon.0) 902 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:21:42.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:41 smithi145 bash[79878]: cluster 2024-03-07T03:21:40.690146+0000 mgr.y (mgr.34107) 283 : cluster 0 pgmap v555: 361 pgs: 43 active+undersized+degraded, 91 active+undersized, 227 active+clean; 3.6 MiB data, 481 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 586 KiB/s wr, 58 op/s; 59/504 objects degraded (11.706%) 2024-03-07T03:21:42.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:41 smithi145 bash[79878]: audit 2024-03-07T03:21:41.197428+0000 mon.a (mon.0) 903 : audit 1 from='osd.2 ' entity='osd.2' 2024-03-07T03:21:42.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:41 smithi145 bash[82329]: audit 2024-03-07T03:21:40.636378+0000 mon.a (mon.0) 900 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-07T03:21:42.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:41 smithi145 bash[82329]: cluster 2024-03-07T03:21:40.645757+0000 mon.a (mon.0) 901 : cluster 0 osdmap e635: 8 total, 7 up, 8 in 2024-03-07T03:21:42.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:41 smithi145 bash[82329]: audit 2024-03-07T03:21:40.647212+0000 mon.c (mon.2) 24 : audit 1 from='osd.2 [v2:172.21.15.145:6818/3198195141,v1:172.21.15.145:6819/3198195141]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:21:42.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:41 smithi145 bash[82329]: audit 2024-03-07T03:21:40.649213+0000 mon.a (mon.0) 902 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:21:42.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:41 smithi145 bash[82329]: cluster 2024-03-07T03:21:40.690146+0000 mgr.y (mgr.34107) 283 : cluster 0 pgmap v555: 361 pgs: 43 active+undersized+degraded, 91 active+undersized, 227 active+clean; 3.6 MiB data, 481 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 586 KiB/s wr, 58 op/s; 59/504 objects degraded (11.706%) 2024-03-07T03:21:42.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:41 smithi145 bash[82329]: audit 2024-03-07T03:21:41.197428+0000 mon.a (mon.0) 903 : audit 1 from='osd.2 ' entity='osd.2' 2024-03-07T03:21:42.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:41 smithi189 bash[66969]: audit 2024-03-07T03:21:40.636378+0000 mon.a (mon.0) 900 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-07T03:21:42.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:41 smithi189 bash[66969]: cluster 2024-03-07T03:21:40.645757+0000 mon.a (mon.0) 901 : cluster 0 osdmap e635: 8 total, 7 up, 8 in 2024-03-07T03:21:42.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:41 smithi189 bash[66969]: audit 2024-03-07T03:21:40.647212+0000 mon.c (mon.2) 24 : audit 1 from='osd.2 [v2:172.21.15.145:6818/3198195141,v1:172.21.15.145:6819/3198195141]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:21:42.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:41 smithi189 bash[66969]: audit 2024-03-07T03:21:40.649213+0000 mon.a (mon.0) 902 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:21:42.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:41 smithi189 bash[66969]: cluster 2024-03-07T03:21:40.690146+0000 mgr.y (mgr.34107) 283 : cluster 0 pgmap v555: 361 pgs: 43 active+undersized+degraded, 91 active+undersized, 227 active+clean; 3.6 MiB data, 481 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 586 KiB/s wr, 58 op/s; 59/504 objects degraded (11.706%) 2024-03-07T03:21:42.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:41 smithi189 bash[66969]: audit 2024-03-07T03:21:41.197428+0000 mon.a (mon.0) 903 : audit 1 from='osd.2 ' entity='osd.2' 2024-03-07T03:21:42.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[79878]: debug 2024-03-07T03:21:42.194+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:21:42 smithi189 bash[59814]: ts=2024-03-07T03:21:42.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:21:43.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:21:42] "GET /metrics HTTP/1.1" 200 55701 "" "Prometheus/2.43.0" 2024-03-07T03:21:43.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[82329]: cluster 2024-03-07T03:21:42.195627+0000 mon.a (mon.0) 904 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:43.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[82329]: cluster 2024-03-07T03:21:42.195712+0000 mon.a (mon.0) 905 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:21:43.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[82329]: cluster 2024-03-07T03:21:42.205864+0000 mon.a (mon.0) 906 : cluster 1 osd.2 [v2:172.21.15.145:6818/3198195141,v1:172.21.15.145:6819/3198195141] boot 2024-03-07T03:21:43.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[82329]: cluster 2024-03-07T03:21:42.205941+0000 mon.a (mon.0) 907 : cluster 0 osdmap e636: 8 total, 8 up, 8 in 2024-03-07T03:21:43.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[82329]: audit 2024-03-07T03:21:42.206301+0000 mon.a (mon.0) 908 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:21:43.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[82329]: audit 2024-03-07T03:21:42.488835+0000 mon.a (mon.0) 909 : audit 0 from='client.? 172.21.15.145:0/212685263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:43.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[79878]: cluster 2024-03-07T03:21:42.195627+0000 mon.a (mon.0) 904 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:43.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[79878]: cluster 2024-03-07T03:21:42.195712+0000 mon.a (mon.0) 905 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:21:43.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[79878]: cluster 2024-03-07T03:21:42.205864+0000 mon.a (mon.0) 906 : cluster 1 osd.2 [v2:172.21.15.145:6818/3198195141,v1:172.21.15.145:6819/3198195141] boot 2024-03-07T03:21:43.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[79878]: cluster 2024-03-07T03:21:42.205941+0000 mon.a (mon.0) 907 : cluster 0 osdmap e636: 8 total, 8 up, 8 in 2024-03-07T03:21:43.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[79878]: audit 2024-03-07T03:21:42.206301+0000 mon.a (mon.0) 908 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:21:43.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:42 smithi145 bash[79878]: audit 2024-03-07T03:21:42.488835+0000 mon.a (mon.0) 909 : audit 0 from='client.? 172.21.15.145:0/212685263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:42 smithi189 bash[66969]: cluster 2024-03-07T03:21:42.195627+0000 mon.a (mon.0) 904 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:42 smithi189 bash[66969]: cluster 2024-03-07T03:21:42.195712+0000 mon.a (mon.0) 905 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:21:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:42 smithi189 bash[66969]: cluster 2024-03-07T03:21:42.205864+0000 mon.a (mon.0) 906 : cluster 1 osd.2 [v2:172.21.15.145:6818/3198195141,v1:172.21.15.145:6819/3198195141] boot 2024-03-07T03:21:43.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:42 smithi189 bash[66969]: cluster 2024-03-07T03:21:42.205941+0000 mon.a (mon.0) 907 : cluster 0 osdmap e636: 8 total, 8 up, 8 in 2024-03-07T03:21:43.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:42 smithi189 bash[66969]: audit 2024-03-07T03:21:42.206301+0000 mon.a (mon.0) 908 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:21:43.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:42 smithi189 bash[66969]: audit 2024-03-07T03:21:42.488835+0000 mon.a (mon.0) 909 : audit 0 from='client.? 172.21.15.145:0/212685263' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:43 smithi145 bash[79878]: cluster 2024-03-07T03:21:42.691491+0000 mgr.y (mgr.34107) 284 : cluster 0 pgmap v557: 361 pgs: 43 active+undersized+degraded, 91 active+undersized, 227 active+clean; 3.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 514 KiB/s wr, 33 op/s; 59/504 objects degraded (11.706%) 2024-03-07T03:21:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:43 smithi145 bash[79878]: cluster 2024-03-07T03:21:43.207018+0000 mon.a (mon.0) 910 : cluster 0 osdmap e637: 8 total, 8 up, 8 in 2024-03-07T03:21:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:43 smithi145 bash[82329]: cluster 2024-03-07T03:21:42.691491+0000 mgr.y (mgr.34107) 284 : cluster 0 pgmap v557: 361 pgs: 43 active+undersized+degraded, 91 active+undersized, 227 active+clean; 3.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 514 KiB/s wr, 33 op/s; 59/504 objects degraded (11.706%) 2024-03-07T03:21:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:43 smithi145 bash[82329]: cluster 2024-03-07T03:21:43.207018+0000 mon.a (mon.0) 910 : cluster 0 osdmap e637: 8 total, 8 up, 8 in 2024-03-07T03:21:44.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:43 smithi189 bash[66969]: cluster 2024-03-07T03:21:42.691491+0000 mgr.y (mgr.34107) 284 : cluster 0 pgmap v557: 361 pgs: 43 active+undersized+degraded, 91 active+undersized, 227 active+clean; 3.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 514 KiB/s wr, 33 op/s; 59/504 objects degraded (11.706%) 2024-03-07T03:21:44.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:43 smithi189 bash[66969]: cluster 2024-03-07T03:21:43.207018+0000 mon.a (mon.0) 910 : cluster 0 osdmap e637: 8 total, 8 up, 8 in 2024-03-07T03:21:45.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:44 smithi145 bash[79878]: cluster 2024-03-07T03:21:44.747041+0000 mon.a (mon.0) 911 : cluster 3 Health check update: Degraded data redundancy: 29/507 objects degraded (5.720%), 21 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:45.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:44 smithi145 bash[82329]: cluster 2024-03-07T03:21:44.747041+0000 mon.a (mon.0) 911 : cluster 3 Health check update: Degraded data redundancy: 29/507 objects degraded (5.720%), 21 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:44 smithi189 bash[66969]: cluster 2024-03-07T03:21:44.747041+0000 mon.a (mon.0) 911 : cluster 3 Health check update: Degraded data redundancy: 29/507 objects degraded (5.720%), 21 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:46.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:45 smithi145 bash[79878]: cluster 2024-03-07T03:21:44.694166+0000 mgr.y (mgr.34107) 285 : cluster 0 pgmap v559: 361 pgs: 21 active+undersized+degraded, 49 active+undersized, 291 active+clean; 7.6 MiB data, 577 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 682 KiB/s wr, 1 op/s; 29/507 objects degraded (5.720%); 0 B/s, 1 objects/s recovering 2024-03-07T03:21:46.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:45 smithi145 bash[79878]: audit 2024-03-07T03:21:44.842688+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.145:0/604463503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:46.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:45 smithi145 bash[82329]: cluster 2024-03-07T03:21:44.694166+0000 mgr.y (mgr.34107) 285 : cluster 0 pgmap v559: 361 pgs: 21 active+undersized+degraded, 49 active+undersized, 291 active+clean; 7.6 MiB data, 577 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 682 KiB/s wr, 1 op/s; 29/507 objects degraded (5.720%); 0 B/s, 1 objects/s recovering 2024-03-07T03:21:46.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:45 smithi145 bash[82329]: audit 2024-03-07T03:21:44.842688+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.145:0/604463503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:46.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:45 smithi189 bash[66969]: cluster 2024-03-07T03:21:44.694166+0000 mgr.y (mgr.34107) 285 : cluster 0 pgmap v559: 361 pgs: 21 active+undersized+degraded, 49 active+undersized, 291 active+clean; 7.6 MiB data, 577 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 682 KiB/s wr, 1 op/s; 29/507 objects degraded (5.720%); 0 B/s, 1 objects/s recovering 2024-03-07T03:21:46.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:45 smithi189 bash[66969]: audit 2024-03-07T03:21:44.842688+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.145:0/604463503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:47 smithi145 bash[79878]: cluster 2024-03-07T03:21:46.695861+0000 mgr.y (mgr.34107) 286 : cluster 0 pgmap v560: 361 pgs: 3 active+undersized+degraded, 12 active+undersized, 346 active+clean; 8.6 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 846 KiB/s wr, 2 op/s; 4/507 objects degraded (0.789%); 677 KiB/s, 1 objects/s recovering 2024-03-07T03:21:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:47 smithi145 bash[79878]: audit 2024-03-07T03:21:47.182997+0000 mon.a (mon.0) 912 : audit 0 from='client.? 172.21.15.145:0/3729058011' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:47 smithi145 bash[79878]: audit 2024-03-07T03:21:47.648324+0000 mon.a (mon.0) 913 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:48.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:47 smithi145 bash[79878]: audit 2024-03-07T03:21:47.651048+0000 mon.a (mon.0) 914 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:48.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:47 smithi145 bash[82329]: cluster 2024-03-07T03:21:46.695861+0000 mgr.y (mgr.34107) 286 : cluster 0 pgmap v560: 361 pgs: 3 active+undersized+degraded, 12 active+undersized, 346 active+clean; 8.6 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 846 KiB/s wr, 2 op/s; 4/507 objects degraded (0.789%); 677 KiB/s, 1 objects/s recovering 2024-03-07T03:21:48.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:47 smithi145 bash[82329]: audit 2024-03-07T03:21:47.182997+0000 mon.a (mon.0) 912 : audit 0 from='client.? 172.21.15.145:0/3729058011' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:48.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:47 smithi145 bash[82329]: audit 2024-03-07T03:21:47.648324+0000 mon.a (mon.0) 913 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:48.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:47 smithi145 bash[82329]: audit 2024-03-07T03:21:47.651048+0000 mon.a (mon.0) 914 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:47 smithi189 bash[66969]: cluster 2024-03-07T03:21:46.695861+0000 mgr.y (mgr.34107) 286 : cluster 0 pgmap v560: 361 pgs: 3 active+undersized+degraded, 12 active+undersized, 346 active+clean; 8.6 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 845 B/s rd, 846 KiB/s wr, 2 op/s; 4/507 objects degraded (0.789%); 677 KiB/s, 1 objects/s recovering 2024-03-07T03:21:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:47 smithi189 bash[66969]: audit 2024-03-07T03:21:47.182997+0000 mon.a (mon.0) 912 : audit 0 from='client.? 172.21.15.145:0/3729058011' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:47 smithi189 bash[66969]: audit 2024-03-07T03:21:47.648324+0000 mon.a (mon.0) 913 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:47 smithi189 bash[66969]: audit 2024-03-07T03:21:47.651048+0000 mon.a (mon.0) 914 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:49.177 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:48 smithi145 bash[79878]: cluster 2024-03-07T03:21:48.806306+0000 mon.a (mon.0) 915 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/507 objects degraded (0.789%), 3 pgs degraded) 2024-03-07T03:21:49.177 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:48 smithi145 bash[82329]: cluster 2024-03-07T03:21:48.806306+0000 mon.a (mon.0) 915 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/507 objects degraded (0.789%), 3 pgs degraded) 2024-03-07T03:21:49.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:48 smithi189 bash[66969]: cluster 2024-03-07T03:21:48.806306+0000 mon.a (mon.0) 915 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/507 objects degraded (0.789%), 3 pgs degraded) 2024-03-07T03:21:50.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[82329]: cluster 2024-03-07T03:21:48.697564+0000 mgr.y (mgr.34107) 287 : cluster 0 pgmap v561: 361 pgs: 361 active+clean; 14 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.2 MiB/s wr, 4 op/s; 512 KiB/s, 0 keys/s, 1 objects/s recovering 2024-03-07T03:21:50.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[82329]: audit 2024-03-07T03:21:49.542992+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.145:0/2118255567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:50.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[82329]: audit 2024-03-07T03:21:49.796976+0000 mon.a (mon.0) 916 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:50.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[82329]: audit 2024-03-07T03:21:49.810330+0000 mon.a (mon.0) 917 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:50.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[82329]: audit 2024-03-07T03:21:49.813203+0000 mon.a (mon.0) 918 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:50.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[82329]: audit 2024-03-07T03:21:49.815103+0000 mon.a (mon.0) 919 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:21:50.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[82329]: audit 2024-03-07T03:21:49.825100+0000 mon.a (mon.0) 920 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:50.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[79878]: cluster 2024-03-07T03:21:48.697564+0000 mgr.y (mgr.34107) 287 : cluster 0 pgmap v561: 361 pgs: 361 active+clean; 14 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.2 MiB/s wr, 4 op/s; 512 KiB/s, 0 keys/s, 1 objects/s recovering 2024-03-07T03:21:50.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[79878]: audit 2024-03-07T03:21:49.542992+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.145:0/2118255567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:50.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[79878]: audit 2024-03-07T03:21:49.796976+0000 mon.a (mon.0) 916 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:50.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[79878]: audit 2024-03-07T03:21:49.810330+0000 mon.a (mon.0) 917 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:50.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[79878]: audit 2024-03-07T03:21:49.813203+0000 mon.a (mon.0) 918 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:50.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[79878]: audit 2024-03-07T03:21:49.815103+0000 mon.a (mon.0) 919 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:21:50.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:49 smithi145 bash[79878]: audit 2024-03-07T03:21:49.825100+0000 mon.a (mon.0) 920 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:50.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:49 smithi189 bash[66969]: cluster 2024-03-07T03:21:48.697564+0000 mgr.y (mgr.34107) 287 : cluster 0 pgmap v561: 361 pgs: 361 active+clean; 14 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.2 MiB/s wr, 4 op/s; 512 KiB/s, 0 keys/s, 1 objects/s recovering 2024-03-07T03:21:50.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:49 smithi189 bash[66969]: audit 2024-03-07T03:21:49.542992+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.145:0/2118255567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:50.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:49 smithi189 bash[66969]: audit 2024-03-07T03:21:49.796976+0000 mon.a (mon.0) 916 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:49 smithi189 bash[66969]: audit 2024-03-07T03:21:49.810330+0000 mon.a (mon.0) 917 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:49 smithi189 bash[66969]: audit 2024-03-07T03:21:49.813203+0000 mon.a (mon.0) 918 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:49 smithi189 bash[66969]: audit 2024-03-07T03:21:49.815103+0000 mon.a (mon.0) 919 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:21:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:49 smithi189 bash[66969]: audit 2024-03-07T03:21:49.825100+0000 mon.a (mon.0) 920 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:51.285 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:50 smithi145 bash[82329]: audit 2024-03-07T03:21:49.924971+0000 mon.a (mon.0) 921 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:51.285 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:50 smithi145 bash[82329]: audit 2024-03-07T03:21:49.929451+0000 mon.a (mon.0) 922 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:51.285 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:50 smithi145 bash[82329]: audit 2024-03-07T03:21:49.933005+0000 mon.a (mon.0) 923 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:51.285 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:50 smithi145 bash[82329]: audit 2024-03-07T03:21:49.935508+0000 mon.a (mon.0) 924 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:51.285 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:50 smithi145 bash[82329]: audit 2024-03-07T03:21:49.939029+0000 mon.a (mon.0) 925 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:21:51.285 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:50 smithi145 bash[79878]: audit 2024-03-07T03:21:49.924971+0000 mon.a (mon.0) 921 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:51.285 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:50 smithi145 bash[79878]: audit 2024-03-07T03:21:49.929451+0000 mon.a (mon.0) 922 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:51.285 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:50 smithi145 bash[79878]: audit 2024-03-07T03:21:49.933005+0000 mon.a (mon.0) 923 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:51.285 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:50 smithi145 bash[79878]: audit 2024-03-07T03:21:49.935508+0000 mon.a (mon.0) 924 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:51.285 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:50 smithi145 bash[79878]: audit 2024-03-07T03:21:49.939029+0000 mon.a (mon.0) 925 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:21:51.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:50 smithi189 bash[66969]: audit 2024-03-07T03:21:49.924971+0000 mon.a (mon.0) 921 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:51.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:50 smithi189 bash[66969]: audit 2024-03-07T03:21:49.929451+0000 mon.a (mon.0) 922 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:51.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:50 smithi189 bash[66969]: audit 2024-03-07T03:21:49.933005+0000 mon.a (mon.0) 923 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:51.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:50 smithi189 bash[66969]: audit 2024-03-07T03:21:49.935508+0000 mon.a (mon.0) 924 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:51.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:50 smithi189 bash[66969]: audit 2024-03-07T03:21:49.939029+0000 mon.a (mon.0) 925 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:21:52.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:51 smithi145 bash[79878]: audit 2024-03-07T03:21:49.939361+0000 mgr.y (mgr.34107) 288 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:21:52.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:51 smithi145 bash[79878]: cephadm 2024-03-07T03:21:49.942729+0000 mgr.y (mgr.34107) 289 : cephadm 1 Upgrade: osd.3 is safe to restart 2024-03-07T03:21:52.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:51 smithi145 bash[79878]: cluster 2024-03-07T03:21:50.698496+0000 mgr.y (mgr.34107) 290 : cluster 0 pgmap v562: 361 pgs: 361 active+clean; 14 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1.2 MiB/s wr, 4 op/s; 482 KiB/s, 0 keys/s, 1 objects/s recovering 2024-03-07T03:21:52.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:51 smithi145 bash[82329]: audit 2024-03-07T03:21:49.939361+0000 mgr.y (mgr.34107) 288 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:21:52.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:51 smithi145 bash[82329]: cephadm 2024-03-07T03:21:49.942729+0000 mgr.y (mgr.34107) 289 : cephadm 1 Upgrade: osd.3 is safe to restart 2024-03-07T03:21:52.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:51 smithi145 bash[82329]: cluster 2024-03-07T03:21:50.698496+0000 mgr.y (mgr.34107) 290 : cluster 0 pgmap v562: 361 pgs: 361 active+clean; 14 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1.2 MiB/s wr, 4 op/s; 482 KiB/s, 0 keys/s, 1 objects/s recovering 2024-03-07T03:21:52.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:51 smithi189 bash[66969]: audit 2024-03-07T03:21:49.939361+0000 mgr.y (mgr.34107) 288 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:21:52.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:51 smithi189 bash[66969]: cephadm 2024-03-07T03:21:49.942729+0000 mgr.y (mgr.34107) 289 : cephadm 1 Upgrade: osd.3 is safe to restart 2024-03-07T03:21:52.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:51 smithi189 bash[66969]: cluster 2024-03-07T03:21:50.698496+0000 mgr.y (mgr.34107) 290 : cluster 0 pgmap v562: 361 pgs: 361 active+clean; 14 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 1.2 MiB/s wr, 4 op/s; 482 KiB/s, 0 keys/s, 1 objects/s recovering 2024-03-07T03:21:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:21:52 smithi189 bash[59814]: ts=2024-03-07T03:21:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:21:52.781 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.RemoveFullTry (20890 ms) 2024-03-07T03:21:52.781 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.RemoveFullTryDataPool 2024-03-07T03:21:53.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:21:52] "GET /metrics HTTP/1.1" 200 55776 "" "Prometheus/2.43.0" 2024-03-07T03:21:53.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[79878]: audit 2024-03-07T03:21:51.884764+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.145:0/2296688666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:53.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[79878]: cluster 2024-03-07T03:21:52.367358+0000 mon.a (mon.0) 926 : cluster 3 pool 'test-librbd-smithi145-68337-23' is full (reached quota's max_bytes: 10 MiB) 2024-03-07T03:21:53.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[79878]: cluster 2024-03-07T03:21:52.367874+0000 mon.a (mon.0) 927 : cluster 3 Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-07T03:21:53.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[79878]: cluster 2024-03-07T03:21:52.381458+0000 mon.a (mon.0) 928 : cluster 0 osdmap e638: 8 total, 8 up, 8 in 2024-03-07T03:21:53.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[79878]: audit 2024-03-07T03:21:52.418544+0000 mon.a (mon.0) 929 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:53.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[79878]: audit 2024-03-07T03:21:52.421239+0000 mon.a (mon.0) 930 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:21:53.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[79878]: audit 2024-03-07T03:21:52.422161+0000 mon.a (mon.0) 931 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:53.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[82329]: audit 2024-03-07T03:21:51.884764+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.145:0/2296688666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:53.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[82329]: cluster 2024-03-07T03:21:52.367358+0000 mon.a (mon.0) 926 : cluster 3 pool 'test-librbd-smithi145-68337-23' is full (reached quota's max_bytes: 10 MiB) 2024-03-07T03:21:53.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[82329]: cluster 2024-03-07T03:21:52.367874+0000 mon.a (mon.0) 927 : cluster 3 Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-07T03:21:53.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[82329]: cluster 2024-03-07T03:21:52.381458+0000 mon.a (mon.0) 928 : cluster 0 osdmap e638: 8 total, 8 up, 8 in 2024-03-07T03:21:53.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[82329]: audit 2024-03-07T03:21:52.418544+0000 mon.a (mon.0) 929 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:53.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[82329]: audit 2024-03-07T03:21:52.421239+0000 mon.a (mon.0) 930 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:21:53.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:53 smithi145 bash[82329]: audit 2024-03-07T03:21:52.422161+0000 mon.a (mon.0) 931 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:53.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:53 smithi189 bash[66969]: audit 2024-03-07T03:21:51.884764+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.145:0/2296688666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:53.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:53 smithi189 bash[66969]: cluster 2024-03-07T03:21:52.367358+0000 mon.a (mon.0) 926 : cluster 3 pool 'test-librbd-smithi145-68337-23' is full (reached quota's max_bytes: 10 MiB) 2024-03-07T03:21:53.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:53 smithi189 bash[66969]: cluster 2024-03-07T03:21:52.367874+0000 mon.a (mon.0) 927 : cluster 3 Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-07T03:21:53.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:53 smithi189 bash[66969]: cluster 2024-03-07T03:21:52.381458+0000 mon.a (mon.0) 928 : cluster 0 osdmap e638: 8 total, 8 up, 8 in 2024-03-07T03:21:53.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:53 smithi189 bash[66969]: audit 2024-03-07T03:21:52.418544+0000 mon.a (mon.0) 929 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:53.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:53 smithi189 bash[66969]: audit 2024-03-07T03:21:52.421239+0000 mon.a (mon.0) 930 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:21:53.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:53 smithi189 bash[66969]: audit 2024-03-07T03:21:52.422161+0000 mon.a (mon.0) 931 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:54.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[79878]: cephadm 2024-03-07T03:21:52.413070+0000 mgr.y (mgr.34107) 291 : cephadm 1 Upgrade: Updating osd.3 2024-03-07T03:21:54.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[79878]: cephadm 2024-03-07T03:21:52.426071+0000 mgr.y (mgr.34107) 292 : cephadm 1 Deploying daemon osd.3 on smithi145 2024-03-07T03:21:54.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[79878]: cluster 2024-03-07T03:21:52.699159+0000 mgr.y (mgr.34107) 293 : cluster 0 pgmap v564: 361 pgs: 361 active+clean; 14 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.1 MiB/s wr, 3 op/s; 431 KiB/s, 0 keys/s, 1 objects/s recovering 2024-03-07T03:21:54.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[79878]: cluster 2024-03-07T03:21:53.433161+0000 mon.a (mon.0) 932 : cluster 0 osdmap e639: 8 total, 8 up, 8 in 2024-03-07T03:21:54.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[79878]: audit 2024-03-07T03:21:53.439476+0000 mon.c (mon.2) 28 : audit 1 from='client.? 172.21.15.145:0/2309861629' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:54.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[79878]: audit 2024-03-07T03:21:53.441619+0000 mon.a (mon.0) 933 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:54.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[79878]: audit 2024-03-07T03:21:53.712824+0000 mon.a (mon.0) 934 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:54.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[79878]: audit 2024-03-07T03:21:54.233740+0000 mon.a (mon.0) 935 : audit 0 from='client.? 172.21.15.145:0/2143149753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:54.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[82329]: cephadm 2024-03-07T03:21:52.413070+0000 mgr.y (mgr.34107) 291 : cephadm 1 Upgrade: Updating osd.3 2024-03-07T03:21:54.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[82329]: cephadm 2024-03-07T03:21:52.426071+0000 mgr.y (mgr.34107) 292 : cephadm 1 Deploying daemon osd.3 on smithi145 2024-03-07T03:21:54.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[82329]: cluster 2024-03-07T03:21:52.699159+0000 mgr.y (mgr.34107) 293 : cluster 0 pgmap v564: 361 pgs: 361 active+clean; 14 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.1 MiB/s wr, 3 op/s; 431 KiB/s, 0 keys/s, 1 objects/s recovering 2024-03-07T03:21:54.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[82329]: cluster 2024-03-07T03:21:53.433161+0000 mon.a (mon.0) 932 : cluster 0 osdmap e639: 8 total, 8 up, 8 in 2024-03-07T03:21:54.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[82329]: audit 2024-03-07T03:21:53.439476+0000 mon.c (mon.2) 28 : audit 1 from='client.? 172.21.15.145:0/2309861629' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:54.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[82329]: audit 2024-03-07T03:21:53.441619+0000 mon.a (mon.0) 933 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:54.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[82329]: audit 2024-03-07T03:21:53.712824+0000 mon.a (mon.0) 934 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:54.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:54 smithi145 bash[82329]: audit 2024-03-07T03:21:54.233740+0000 mon.a (mon.0) 935 : audit 0 from='client.? 172.21.15.145:0/2143149753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:54.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:54 smithi189 bash[66969]: cephadm 2024-03-07T03:21:52.413070+0000 mgr.y (mgr.34107) 291 : cephadm 1 Upgrade: Updating osd.3 2024-03-07T03:21:54.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:54 smithi189 bash[66969]: cephadm 2024-03-07T03:21:52.426071+0000 mgr.y (mgr.34107) 292 : cephadm 1 Deploying daemon osd.3 on smithi145 2024-03-07T03:21:54.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:54 smithi189 bash[66969]: cluster 2024-03-07T03:21:52.699159+0000 mgr.y (mgr.34107) 293 : cluster 0 pgmap v564: 361 pgs: 361 active+clean; 14 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.1 MiB/s wr, 3 op/s; 431 KiB/s, 0 keys/s, 1 objects/s recovering 2024-03-07T03:21:54.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:54 smithi189 bash[66969]: cluster 2024-03-07T03:21:53.433161+0000 mon.a (mon.0) 932 : cluster 0 osdmap e639: 8 total, 8 up, 8 in 2024-03-07T03:21:54.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:54 smithi189 bash[66969]: audit 2024-03-07T03:21:53.439476+0000 mon.c (mon.2) 28 : audit 1 from='client.? 172.21.15.145:0/2309861629' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:54.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:54 smithi189 bash[66969]: audit 2024-03-07T03:21:53.441619+0000 mon.a (mon.0) 933 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:54.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:54 smithi189 bash[66969]: audit 2024-03-07T03:21:53.712824+0000 mon.a (mon.0) 934 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:21:54.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:54 smithi189 bash[66969]: audit 2024-03-07T03:21:54.233740+0000 mon.a (mon.0) 935 : audit 0 from='client.? 172.21.15.145:0/2143149753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:55.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:55 smithi145 bash[79878]: audit 2024-03-07T03:21:54.427430+0000 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:55.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:55 smithi145 bash[79878]: cluster 2024-03-07T03:21:54.438502+0000 mon.a (mon.0) 937 : cluster 0 osdmap e640: 8 total, 8 up, 8 in 2024-03-07T03:21:55.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:55 smithi145 bash[79878]: cluster 2024-03-07T03:21:54.700027+0000 mgr.y (mgr.34107) 294 : cluster 0 pgmap v567: 393 pgs: 21 creating+peering, 11 unknown, 361 active+clean; 4.6 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 513 KiB/s wr, 25 op/s 2024-03-07T03:21:55.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:55 smithi145 bash[82329]: audit 2024-03-07T03:21:54.427430+0000 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:55.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:55 smithi145 bash[82329]: cluster 2024-03-07T03:21:54.438502+0000 mon.a (mon.0) 937 : cluster 0 osdmap e640: 8 total, 8 up, 8 in 2024-03-07T03:21:55.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:55 smithi145 bash[82329]: cluster 2024-03-07T03:21:54.700027+0000 mgr.y (mgr.34107) 294 : cluster 0 pgmap v567: 393 pgs: 21 creating+peering, 11 unknown, 361 active+clean; 4.6 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 513 KiB/s wr, 25 op/s 2024-03-07T03:21:55.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:55 smithi189 bash[66969]: audit 2024-03-07T03:21:54.427430+0000 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:55.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:55 smithi189 bash[66969]: cluster 2024-03-07T03:21:54.438502+0000 mon.a (mon.0) 937 : cluster 0 osdmap e640: 8 total, 8 up, 8 in 2024-03-07T03:21:55.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:55 smithi189 bash[66969]: cluster 2024-03-07T03:21:54.700027+0000 mgr.y (mgr.34107) 294 : cluster 0 pgmap v567: 393 pgs: 21 creating+peering, 11 unknown, 361 active+clean; 4.6 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 513 KiB/s wr, 25 op/s 2024-03-07T03:21:56.502 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:21:56.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:56 smithi145 bash[72238]: [07/Mar/2024:03:21:56] ENGINE Bus STOPPING 2024-03-07T03:21:56.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:56 smithi145 bash[72238]: [07/Mar/2024:03:21:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:21:56.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:56 smithi145 bash[72238]: [07/Mar/2024:03:21:56] ENGINE Bus STOPPED 2024-03-07T03:21:56.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:56 smithi145 bash[72238]: [07/Mar/2024:03:21:56] ENGINE Bus STARTING 2024-03-07T03:21:56.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:56 smithi145 bash[72238]: [07/Mar/2024:03:21:56] ENGINE Serving on http://:::9283 2024-03-07T03:21:56.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:56 smithi145 bash[72238]: [07/Mar/2024:03:21:56] ENGINE Bus STARTED 2024-03-07T03:21:56.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:56 smithi145 bash[79878]: cluster 2024-03-07T03:21:55.451794+0000 mon.a (mon.0) 938 : cluster 0 osdmap e641: 8 total, 8 up, 8 in 2024-03-07T03:21:56.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:56 smithi145 bash[79878]: audit 2024-03-07T03:21:55.453136+0000 mon.c (mon.2) 29 : audit 1 from='client.? 172.21.15.145:0/1142309926' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:56.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:56 smithi145 bash[79878]: audit 2024-03-07T03:21:55.454848+0000 mon.a (mon.0) 939 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:56.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:56 smithi145 bash[82329]: cluster 2024-03-07T03:21:55.451794+0000 mon.a (mon.0) 938 : cluster 0 osdmap e641: 8 total, 8 up, 8 in 2024-03-07T03:21:56.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:56 smithi145 bash[82329]: audit 2024-03-07T03:21:55.453136+0000 mon.c (mon.2) 29 : audit 1 from='client.? 172.21.15.145:0/1142309926' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:56.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:56 smithi145 bash[82329]: audit 2024-03-07T03:21:55.454848+0000 mon.a (mon.0) 939 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:56.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:56 smithi189 bash[66969]: cluster 2024-03-07T03:21:55.451794+0000 mon.a (mon.0) 938 : cluster 0 osdmap e641: 8 total, 8 up, 8 in 2024-03-07T03:21:56.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:56 smithi189 bash[66969]: audit 2024-03-07T03:21:55.453136+0000 mon.c (mon.2) 29 : audit 1 from='client.? 172.21.15.145:0/1142309926' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:56.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:56 smithi189 bash[66969]: audit 2024-03-07T03:21:55.454848+0000 mon.a (mon.0) 939 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:57.457 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:57 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:57.458 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:21:57 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:57.458 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:21:57 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:57.458 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:21:57 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:57.459 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:21:57 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:57.459 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:57 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:57.459 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:21:57 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:57.460 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:21:57 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:57.460 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:21:57 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:57.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[79878]: audit 2024-03-07T03:21:56.442757+0000 mon.a (mon.0) 940 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:57.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[79878]: cluster 2024-03-07T03:21:56.455350+0000 mon.a (mon.0) 941 : cluster 0 osdmap e642: 8 total, 8 up, 8 in 2024-03-07T03:21:57.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[79878]: audit 2024-03-07T03:21:56.514406+0000 mon.a (mon.0) 942 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' 2024-03-07T03:21:57.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[79878]: audit 2024-03-07T03:21:56.617378+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.145:0/3822342600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:57.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[79878]: cluster 2024-03-07T03:21:56.701462+0000 mgr.y (mgr.34107) 295 : cluster 0 pgmap v570: 425 pgs: 24 creating+peering, 29 unknown, 372 active+clean; 584 KiB data, 457 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 770 KiB/s wr, 39 op/s 2024-03-07T03:21:57.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[79878]: cluster 2024-03-07T03:21:57.369194+0000 mon.a (mon.0) 943 : cluster 1 pool 'test-librbd-smithi145-68337-23' no longer out of quota; removing NO_QUOTA flag 2024-03-07T03:21:57.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[79878]: cluster 2024-03-07T03:21:57.369505+0000 mon.a (mon.0) 944 : cluster 1 Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-07T03:21:57.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[79878]: cluster 2024-03-07T03:21:57.374743+0000 mon.a (mon.0) 945 : cluster 0 osdmap e643: 8 total, 8 up, 8 in 2024-03-07T03:21:57.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[82329]: audit 2024-03-07T03:21:56.442757+0000 mon.a (mon.0) 940 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:57.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[82329]: cluster 2024-03-07T03:21:56.455350+0000 mon.a (mon.0) 941 : cluster 0 osdmap e642: 8 total, 8 up, 8 in 2024-03-07T03:21:57.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[82329]: audit 2024-03-07T03:21:56.514406+0000 mon.a (mon.0) 942 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' 2024-03-07T03:21:57.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[82329]: audit 2024-03-07T03:21:56.617378+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.145:0/3822342600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:57.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[82329]: cluster 2024-03-07T03:21:56.701462+0000 mgr.y (mgr.34107) 295 : cluster 0 pgmap v570: 425 pgs: 24 creating+peering, 29 unknown, 372 active+clean; 584 KiB data, 457 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 770 KiB/s wr, 39 op/s 2024-03-07T03:21:57.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[82329]: cluster 2024-03-07T03:21:57.369194+0000 mon.a (mon.0) 943 : cluster 1 pool 'test-librbd-smithi145-68337-23' no longer out of quota; removing NO_QUOTA flag 2024-03-07T03:21:57.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[82329]: cluster 2024-03-07T03:21:57.369505+0000 mon.a (mon.0) 944 : cluster 1 Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-07T03:21:57.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[82329]: cluster 2024-03-07T03:21:57.374743+0000 mon.a (mon.0) 945 : cluster 0 osdmap e643: 8 total, 8 up, 8 in 2024-03-07T03:21:57.812 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:21:57 smithi145 systemd[1]: Stopping Ceph osd.3 for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:21:57.812 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[51007]: debug 2024-03-07T03:21:57.605+0000 7f4f6b170700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:21:57.812 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[51007]: debug 2024-03-07T03:21:57.605+0000 7f4f6b170700 -1 osd.3 643 *** Got signal Terminated *** 2024-03-07T03:21:57.812 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:21:57 smithi145 bash[51007]: debug 2024-03-07T03:21:57.605+0000 7f4f6b170700 -1 osd.3 643 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:21:57.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:57 smithi189 bash[66969]: audit 2024-03-07T03:21:56.442757+0000 mon.a (mon.0) 940 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:57.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:57 smithi189 bash[66969]: cluster 2024-03-07T03:21:56.455350+0000 mon.a (mon.0) 941 : cluster 0 osdmap e642: 8 total, 8 up, 8 in 2024-03-07T03:21:57.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:57 smithi189 bash[66969]: audit 2024-03-07T03:21:56.514406+0000 mon.a (mon.0) 942 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' 2024-03-07T03:21:57.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:57 smithi189 bash[66969]: audit 2024-03-07T03:21:56.617378+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.145:0/3822342600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:57.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:57 smithi189 bash[66969]: cluster 2024-03-07T03:21:56.701462+0000 mgr.y (mgr.34107) 295 : cluster 0 pgmap v570: 425 pgs: 24 creating+peering, 29 unknown, 372 active+clean; 584 KiB data, 457 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 770 KiB/s wr, 39 op/s 2024-03-07T03:21:57.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:57 smithi189 bash[66969]: cluster 2024-03-07T03:21:57.369194+0000 mon.a (mon.0) 943 : cluster 1 pool 'test-librbd-smithi145-68337-23' no longer out of quota; removing NO_QUOTA flag 2024-03-07T03:21:57.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:57 smithi189 bash[66969]: cluster 2024-03-07T03:21:57.369505+0000 mon.a (mon.0) 944 : cluster 1 Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-07T03:21:57.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:57 smithi189 bash[66969]: cluster 2024-03-07T03:21:57.374743+0000 mon.a (mon.0) 945 : cluster 0 osdmap e643: 8 total, 8 up, 8 in 2024-03-07T03:21:58.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:58 smithi145 bash[82329]: cluster 2024-03-07T03:21:57.606690+0000 mon.a (mon.0) 946 : cluster 1 osd.3 marked itself down and dead 2024-03-07T03:21:58.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:58 smithi145 bash[82329]: cluster 2024-03-07T03:21:58.373059+0000 mon.a (mon.0) 947 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:21:58.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:58 smithi145 bash[82329]: cluster 2024-03-07T03:21:58.373112+0000 mon.a (mon.0) 948 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:58.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:58 smithi145 bash[82329]: cluster 2024-03-07T03:21:58.384258+0000 mon.a (mon.0) 949 : cluster 0 osdmap e644: 8 total, 7 up, 8 in 2024-03-07T03:21:58.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:58 smithi145 bash[82329]: cluster 2024-03-07T03:21:58.393034+0000 mon.a (mon.0) 950 : cluster 1 osd.3 marked itself dead as of e644 2024-03-07T03:21:58.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:58 smithi145 bash[79878]: debug 2024-03-07T03:21:58.369+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:58.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:58 smithi145 bash[79878]: cluster 2024-03-07T03:21:57.606690+0000 mon.a (mon.0) 946 : cluster 1 osd.3 marked itself down and dead 2024-03-07T03:21:58.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:58 smithi145 bash[79878]: cluster 2024-03-07T03:21:58.373059+0000 mon.a (mon.0) 947 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:21:58.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:58 smithi145 bash[79878]: cluster 2024-03-07T03:21:58.373112+0000 mon.a (mon.0) 948 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:58.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:58 smithi145 bash[79878]: cluster 2024-03-07T03:21:58.384258+0000 mon.a (mon.0) 949 : cluster 0 osdmap e644: 8 total, 7 up, 8 in 2024-03-07T03:21:58.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:58 smithi145 bash[79878]: cluster 2024-03-07T03:21:58.393034+0000 mon.a (mon.0) 950 : cluster 1 osd.3 marked itself dead as of e644 2024-03-07T03:21:58.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:58 smithi189 bash[66969]: cluster 2024-03-07T03:21:57.606690+0000 mon.a (mon.0) 946 : cluster 1 osd.3 marked itself down and dead 2024-03-07T03:21:58.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:58 smithi189 bash[66969]: cluster 2024-03-07T03:21:58.373059+0000 mon.a (mon.0) 947 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:21:58.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:58 smithi189 bash[66969]: cluster 2024-03-07T03:21:58.373112+0000 mon.a (mon.0) 948 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:58.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:58 smithi189 bash[66969]: cluster 2024-03-07T03:21:58.384258+0000 mon.a (mon.0) 949 : cluster 0 osdmap e644: 8 total, 7 up, 8 in 2024-03-07T03:21:58.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:58 smithi189 bash[66969]: cluster 2024-03-07T03:21:58.393034+0000 mon.a (mon.0) 950 : cluster 1 osd.3 marked itself dead as of e644 2024-03-07T03:21:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:59 smithi145 bash[82329]: cluster 2024-03-07T03:21:58.703216+0000 mgr.y (mgr.34107) 296 : cluster 0 pgmap v573: 425 pgs: 100 peering, 1 stale+creating+peering, 25 stale+active+clean, 3 creating+peering, 1 unknown, 295 active+clean; 584 KiB data, 457 MiB used, 715 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2024-03-07T03:21:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:59 smithi145 bash[82329]: audit 2024-03-07T03:21:58.974224+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.145:0/495820767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:59 smithi145 bash[82329]: cluster 2024-03-07T03:21:59.385410+0000 mon.a (mon.0) 951 : cluster 0 osdmap e645: 8 total, 7 up, 8 in 2024-03-07T03:21:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:21:59 smithi145 bash[82329]: audit 2024-03-07T03:21:59.484800+0000 mon.a (mon.0) 952 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi145-68337-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:21:59.810 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:21:59 smithi145 bash[103373]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-3 2024-03-07T03:21:59.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:59 smithi145 bash[79878]: cluster 2024-03-07T03:21:58.703216+0000 mgr.y (mgr.34107) 296 : cluster 0 pgmap v573: 425 pgs: 100 peering, 1 stale+creating+peering, 25 stale+active+clean, 3 creating+peering, 1 unknown, 295 active+clean; 584 KiB data, 457 MiB used, 715 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2024-03-07T03:21:59.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:59 smithi145 bash[79878]: audit 2024-03-07T03:21:58.974224+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.145:0/495820767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:59.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:59 smithi145 bash[79878]: cluster 2024-03-07T03:21:59.385410+0000 mon.a (mon.0) 951 : cluster 0 osdmap e645: 8 total, 7 up, 8 in 2024-03-07T03:21:59.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:21:59 smithi145 bash[79878]: audit 2024-03-07T03:21:59.484800+0000 mon.a (mon.0) 952 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi145-68337-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:21:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:59 smithi189 bash[66969]: cluster 2024-03-07T03:21:58.703216+0000 mgr.y (mgr.34107) 296 : cluster 0 pgmap v573: 425 pgs: 100 peering, 1 stale+creating+peering, 25 stale+active+clean, 3 creating+peering, 1 unknown, 295 active+clean; 584 KiB data, 457 MiB used, 715 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2024-03-07T03:21:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:59 smithi189 bash[66969]: audit 2024-03-07T03:21:58.974224+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.145:0/495820767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:21:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:59 smithi189 bash[66969]: cluster 2024-03-07T03:21:59.385410+0000 mon.a (mon.0) 951 : cluster 0 osdmap e645: 8 total, 7 up, 8 in 2024-03-07T03:21:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:21:59 smithi189 bash[66969]: audit 2024-03-07T03:21:59.484800+0000 mon.a (mon.0) 952 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi145-68337-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:22:00.310 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:21:59 smithi145 bash[103466]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-3 2024-03-07T03:22:00.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:00 smithi145 bash[82329]: cluster 2024-03-07T03:21:59.515746+0000 mon.a (mon.0) 953 : cluster 3 Health check failed: Reduced data availability: 16 pgs inactive, 21 pgs peering (PG_AVAILABILITY) 2024-03-07T03:22:00.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:00 smithi145 bash[82329]: audit 2024-03-07T03:22:00.381070+0000 mon.a (mon.0) 954 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi145-68337-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-07T03:22:00.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:00 smithi145 bash[82329]: cluster 2024-03-07T03:22:00.388446+0000 mon.a (mon.0) 955 : cluster 0 osdmap e646: 8 total, 7 up, 8 in 2024-03-07T03:22:00.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:00 smithi145 bash[79878]: cluster 2024-03-07T03:21:59.515746+0000 mon.a (mon.0) 953 : cluster 3 Health check failed: Reduced data availability: 16 pgs inactive, 21 pgs peering (PG_AVAILABILITY) 2024-03-07T03:22:00.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:00 smithi145 bash[79878]: audit 2024-03-07T03:22:00.381070+0000 mon.a (mon.0) 954 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi145-68337-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-07T03:22:00.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:00 smithi145 bash[79878]: cluster 2024-03-07T03:22:00.388446+0000 mon.a (mon.0) 955 : cluster 0 osdmap e646: 8 total, 7 up, 8 in 2024-03-07T03:22:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:00 smithi189 bash[66969]: cluster 2024-03-07T03:21:59.515746+0000 mon.a (mon.0) 953 : cluster 3 Health check failed: Reduced data availability: 16 pgs inactive, 21 pgs peering (PG_AVAILABILITY) 2024-03-07T03:22:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:00 smithi189 bash[66969]: audit 2024-03-07T03:22:00.381070+0000 mon.a (mon.0) 954 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi145-68337-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-07T03:22:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:00 smithi189 bash[66969]: cluster 2024-03-07T03:22:00.388446+0000 mon.a (mon.0) 955 : cluster 0 osdmap e646: 8 total, 7 up, 8 in 2024-03-07T03:22:01.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:01 smithi145 bash[79878]: cluster 2024-03-07T03:22:00.704626+0000 mgr.y (mgr.34107) 297 : cluster 0 pgmap v576: 425 pgs: 100 peering, 1 stale+creating+peering, 25 stale+active+clean, 3 creating+peering, 1 unknown, 295 active+clean; 584 KiB data, 457 MiB used, 715 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2024-03-07T03:22:01.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:01 smithi145 bash[79878]: audit 2024-03-07T03:22:01.337071+0000 mon.a (mon.0) 956 : audit 0 from='client.? 172.21.15.145:0/1170626125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:01.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:01 smithi145 bash[82329]: cluster 2024-03-07T03:22:00.704626+0000 mgr.y (mgr.34107) 297 : cluster 0 pgmap v576: 425 pgs: 100 peering, 1 stale+creating+peering, 25 stale+active+clean, 3 creating+peering, 1 unknown, 295 active+clean; 584 KiB data, 457 MiB used, 715 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2024-03-07T03:22:01.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:01 smithi145 bash[82329]: audit 2024-03-07T03:22:01.337071+0000 mon.a (mon.0) 956 : audit 0 from='client.? 172.21.15.145:0/1170626125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:01.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:01 smithi189 bash[66969]: cluster 2024-03-07T03:22:00.704626+0000 mgr.y (mgr.34107) 297 : cluster 0 pgmap v576: 425 pgs: 100 peering, 1 stale+creating+peering, 25 stale+active+clean, 3 creating+peering, 1 unknown, 295 active+clean; 584 KiB data, 457 MiB used, 715 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2024-03-07T03:22:01.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:01 smithi189 bash[66969]: audit 2024-03-07T03:22:01.337071+0000 mon.a (mon.0) 956 : audit 0 from='client.? 172.21.15.145:0/1170626125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:02.552 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:22:02 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:02.553 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:02 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:02.553 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:22:02 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:02.554 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:22:02 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:02.554 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:22:02 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:02.554 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:22:02 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:02.554 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:02 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:02.555 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:22:02 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:02.555 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:02 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.3.service: Deactivated successfully. 2024-03-07T03:22:02.555 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:02 smithi145 systemd[1]: Stopped Ceph osd.3 for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:22:02.555 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:02 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:02.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:22:02 smithi189 bash[59814]: ts=2024-03-07T03:22:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:22:02.810 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:02 smithi145 systemd[1]: Started Ceph osd.3 for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:22:02.810 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:22:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:22:02] "GET /metrics HTTP/1.1" 200 60519 "" "Prometheus/2.43.0" 2024-03-07T03:22:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:03 smithi145 bash[79878]: audit 2024-03-07T03:22:02.608999+0000 mon.a (mon.0) 957 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:03 smithi145 bash[79878]: audit 2024-03-07T03:22:02.616560+0000 mon.a (mon.0) 958 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:03 smithi145 bash[79878]: audit 2024-03-07T03:22:02.643880+0000 mon.a (mon.0) 959 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:03 smithi145 bash[79878]: audit 2024-03-07T03:22:02.644995+0000 mon.a (mon.0) 960 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:03 smithi145 bash[79878]: audit 2024-03-07T03:22:02.682860+0000 mon.a (mon.0) 961 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:03 smithi145 bash[79878]: cluster 2024-03-07T03:22:02.706567+0000 mgr.y (mgr.34107) 298 : cluster 0 pgmap v577: 425 pgs: 25 active+undersized+degraded, 56 active+undersized, 100 peering, 244 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s; 32/504 objects degraded (6.349%) 2024-03-07T03:22:03.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:03 smithi145 bash[82329]: audit 2024-03-07T03:22:02.608999+0000 mon.a (mon.0) 957 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:03.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:03 smithi145 bash[82329]: audit 2024-03-07T03:22:02.616560+0000 mon.a (mon.0) 958 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:03.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:03 smithi145 bash[82329]: audit 2024-03-07T03:22:02.643880+0000 mon.a (mon.0) 959 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:03.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:03 smithi145 bash[82329]: audit 2024-03-07T03:22:02.644995+0000 mon.a (mon.0) 960 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:03.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:03 smithi145 bash[82329]: audit 2024-03-07T03:22:02.682860+0000 mon.a (mon.0) 961 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:03.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:03 smithi145 bash[82329]: cluster 2024-03-07T03:22:02.706567+0000 mgr.y (mgr.34107) 298 : cluster 0 pgmap v577: 425 pgs: 25 active+undersized+degraded, 56 active+undersized, 100 peering, 244 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s; 32/504 objects degraded (6.349%) 2024-03-07T03:22:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:03 smithi189 bash[66969]: audit 2024-03-07T03:22:02.608999+0000 mon.a (mon.0) 957 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:04.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:03 smithi189 bash[66969]: audit 2024-03-07T03:22:02.616560+0000 mon.a (mon.0) 958 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:04.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:03 smithi189 bash[66969]: audit 2024-03-07T03:22:02.643880+0000 mon.a (mon.0) 959 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:04.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:03 smithi189 bash[66969]: audit 2024-03-07T03:22:02.644995+0000 mon.a (mon.0) 960 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:04.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:03 smithi189 bash[66969]: audit 2024-03-07T03:22:02.682860+0000 mon.a (mon.0) 961 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:04.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:03 smithi189 bash[66969]: cluster 2024-03-07T03:22:02.706567+0000 mgr.y (mgr.34107) 298 : cluster 0 pgmap v577: 425 pgs: 25 active+undersized+degraded, 56 active+undersized, 100 peering, 244 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s; 32/504 objects degraded (6.349%) 2024-03-07T03:22:04.560 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:04 smithi145 bash[103644]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-07T03:22:04.560 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:04 smithi145 bash[103644]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-07T03:22:04.560 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:04 smithi145 bash[103644]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-07T03:22:04.560 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:04 smithi145 bash[103644]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-07T03:22:04.560 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:04 smithi145 bash[103644]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-07T03:22:04.561 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:04 smithi145 bash[103644]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-07T03:22:04.561 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:04 smithi145 bash[103644]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-07T03:22:04.970 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:04 smithi145 bash[79878]: cluster 2024-03-07T03:22:03.641819+0000 mon.a (mon.0) 962 : cluster 3 Health check failed: Degraded data redundancy: 32/504 objects degraded (6.349%), 25 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:04.970 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:04 smithi145 bash[79878]: audit 2024-03-07T03:22:03.704357+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.145:0/3020659641' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:04.970 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:04 smithi145 bash[82329]: cluster 2024-03-07T03:22:03.641819+0000 mon.a (mon.0) 962 : cluster 3 Health check failed: Degraded data redundancy: 32/504 objects degraded (6.349%), 25 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:04.971 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:04 smithi145 bash[82329]: audit 2024-03-07T03:22:03.704357+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.145:0/3020659641' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:05.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:04 smithi189 bash[66969]: cluster 2024-03-07T03:22:03.641819+0000 mon.a (mon.0) 962 : cluster 3 Health check failed: Degraded data redundancy: 32/504 objects degraded (6.349%), 25 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:05.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:04 smithi189 bash[66969]: audit 2024-03-07T03:22:03.704357+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.145:0/3020659641' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:05.875 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:05 smithi145 bash[79878]: cluster 2024-03-07T03:22:04.709588+0000 mgr.y (mgr.34107) 299 : cluster 0 pgmap v578: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 4.6 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 649 KiB/s wr, 2 op/s; 70/513 objects degraded (13.645%) 2024-03-07T03:22:05.875 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:05 smithi145 bash[82329]: cluster 2024-03-07T03:22:04.709588+0000 mgr.y (mgr.34107) 299 : cluster 0 pgmap v578: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 4.6 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 649 KiB/s wr, 2 op/s; 70/513 objects degraded (13.645%) 2024-03-07T03:22:06.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:05 smithi189 bash[66969]: cluster 2024-03-07T03:22:04.709588+0000 mgr.y (mgr.34107) 299 : cluster 0 pgmap v578: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 4.6 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 649 KiB/s wr, 2 op/s; 70/513 objects degraded (13.645%) 2024-03-07T03:22:06.310 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:05 smithi145 bash[103903]: debug 2024-03-07T03:22:05.941+0000 7ffb02324700 1 -- 172.21.15.145:0/461831826 <== mon.0 v2:172.21.15.145:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5594efce7680 con 0x5594efd22c00 2024-03-07T03:22:06.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:06 smithi145 bash[82329]: cluster 2024-03-07T03:22:05.610160+0000 mon.a (mon.0) 963 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 18 pgs inactive, 23 pgs peering) 2024-03-07T03:22:06.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:06 smithi145 bash[82329]: audit 2024-03-07T03:22:06.063208+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.145:0/3819946706' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:06.810 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:06 smithi145 bash[103903]: debug 2024-03-07T03:22:06.489+0000 7ffb0b4bc7c0 -1 Falling back to public interface 2024-03-07T03:22:06.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:06 smithi145 bash[79878]: cluster 2024-03-07T03:22:05.610160+0000 mon.a (mon.0) 963 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 18 pgs inactive, 23 pgs peering) 2024-03-07T03:22:06.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:06 smithi145 bash[79878]: audit 2024-03-07T03:22:06.063208+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.145:0/3819946706' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:07.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:06 smithi189 bash[66969]: cluster 2024-03-07T03:22:05.610160+0000 mon.a (mon.0) 963 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 18 pgs inactive, 23 pgs peering) 2024-03-07T03:22:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:06 smithi189 bash[66969]: audit 2024-03-07T03:22:06.063208+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.145:0/3819946706' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:08.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:07 smithi145 bash[79878]: cluster 2024-03-07T03:22:06.711990+0000 mgr.y (mgr.34107) 300 : cluster 0 pgmap v579: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 4.6 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 512 KiB/s wr, 1 op/s; 70/513 objects degraded (13.645%) 2024-03-07T03:22:08.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:07 smithi145 bash[82329]: cluster 2024-03-07T03:22:06.711990+0000 mgr.y (mgr.34107) 300 : cluster 0 pgmap v579: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 4.6 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 512 KiB/s wr, 1 op/s; 70/513 objects degraded (13.645%) 2024-03-07T03:22:08.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:07 smithi189 bash[66969]: cluster 2024-03-07T03:22:06.711990+0000 mgr.y (mgr.34107) 300 : cluster 0 pgmap v579: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 4.6 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 512 KiB/s wr, 1 op/s; 70/513 objects degraded (13.645%) 2024-03-07T03:22:09.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:08 smithi145 bash[79878]: audit 2024-03-07T03:22:08.448253+0000 mon.a (mon.0) 964 : audit 0 from='client.? 172.21.15.145:0/4220999288' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:09.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:08 smithi145 bash[79878]: audit 2024-03-07T03:22:08.480758+0000 mon.a (mon.0) 965 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:09.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:08 smithi145 bash[79878]: audit 2024-03-07T03:22:08.488693+0000 mon.a (mon.0) 966 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:09.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:08 smithi145 bash[82329]: audit 2024-03-07T03:22:08.448253+0000 mon.a (mon.0) 964 : audit 0 from='client.? 172.21.15.145:0/4220999288' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:09.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:08 smithi145 bash[82329]: audit 2024-03-07T03:22:08.480758+0000 mon.a (mon.0) 965 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:09.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:08 smithi145 bash[82329]: audit 2024-03-07T03:22:08.488693+0000 mon.a (mon.0) 966 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:09.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:08 smithi189 bash[66969]: audit 2024-03-07T03:22:08.448253+0000 mon.a (mon.0) 964 : audit 0 from='client.? 172.21.15.145:0/4220999288' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:09.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:08 smithi189 bash[66969]: audit 2024-03-07T03:22:08.480758+0000 mon.a (mon.0) 965 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:09.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:08 smithi189 bash[66969]: audit 2024-03-07T03:22:08.488693+0000 mon.a (mon.0) 966 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:10.060 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:09 smithi145 bash[103903]: debug 2024-03-07T03:22:09.701+0000 7ffb0b4bc7c0 -1 osd.3 644 log_to_monitors true 2024-03-07T03:22:10.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:09 smithi145 bash[79878]: cluster 2024-03-07T03:22:08.715389+0000 mgr.y (mgr.34107) 301 : cluster 0 pgmap v580: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 9.6 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 988 KiB/s wr, 2 op/s; 70/519 objects degraded (13.487%) 2024-03-07T03:22:10.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:09 smithi145 bash[79878]: audit 2024-03-07T03:22:09.532605+0000 mon.a (mon.0) 967 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:10.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:09 smithi145 bash[79878]: audit 2024-03-07T03:22:09.538406+0000 mon.a (mon.0) 968 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:10.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:09 smithi145 bash[82329]: cluster 2024-03-07T03:22:08.715389+0000 mgr.y (mgr.34107) 301 : cluster 0 pgmap v580: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 9.6 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 988 KiB/s wr, 2 op/s; 70/519 objects degraded (13.487%) 2024-03-07T03:22:10.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:09 smithi145 bash[82329]: audit 2024-03-07T03:22:09.532605+0000 mon.a (mon.0) 967 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:10.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:09 smithi145 bash[82329]: audit 2024-03-07T03:22:09.538406+0000 mon.a (mon.0) 968 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:10.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:09 smithi189 bash[66969]: cluster 2024-03-07T03:22:08.715389+0000 mgr.y (mgr.34107) 301 : cluster 0 pgmap v580: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 9.6 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 988 KiB/s wr, 2 op/s; 70/519 objects degraded (13.487%) 2024-03-07T03:22:10.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:09 smithi189 bash[66969]: audit 2024-03-07T03:22:09.532605+0000 mon.a (mon.0) 967 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:10.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:09 smithi189 bash[66969]: audit 2024-03-07T03:22:09.538406+0000 mon.a (mon.0) 968 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:11.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:10 smithi145 bash[79878]: cluster 2024-03-07T03:22:09.633933+0000 mon.a (mon.0) 969 : cluster 3 Health check update: Degraded data redundancy: 70/519 objects degraded (13.487%), 46 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:11.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:10 smithi145 bash[79878]: audit 2024-03-07T03:22:09.709330+0000 mon.c (mon.2) 34 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2193661319,v1:172.21.15.145:6827/2193661319]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:22:11.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:10 smithi145 bash[79878]: audit 2024-03-07T03:22:09.709717+0000 mon.a (mon.0) 970 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:22:11.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:10 smithi145 bash[82329]: cluster 2024-03-07T03:22:09.633933+0000 mon.a (mon.0) 969 : cluster 3 Health check update: Degraded data redundancy: 70/519 objects degraded (13.487%), 46 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:11.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:10 smithi145 bash[82329]: audit 2024-03-07T03:22:09.709330+0000 mon.c (mon.2) 34 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2193661319,v1:172.21.15.145:6827/2193661319]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:22:11.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:10 smithi145 bash[82329]: audit 2024-03-07T03:22:09.709717+0000 mon.a (mon.0) 970 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:22:11.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:10 smithi189 bash[66969]: cluster 2024-03-07T03:22:09.633933+0000 mon.a (mon.0) 969 : cluster 3 Health check update: Degraded data redundancy: 70/519 objects degraded (13.487%), 46 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:11.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:10 smithi189 bash[66969]: audit 2024-03-07T03:22:09.709330+0000 mon.c (mon.2) 34 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2193661319,v1:172.21.15.145:6827/2193661319]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:22:11.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:10 smithi189 bash[66969]: audit 2024-03-07T03:22:09.709717+0000 mon.a (mon.0) 970 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:22:11.560 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[103903]: debug 2024-03-07T03:22:11.205+0000 7ffaf890c700 -1 osd.3 644 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-07T03:22:12.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[79878]: audit 2024-03-07T03:22:10.644862+0000 mon.a (mon.0) 971 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-07T03:22:12.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[79878]: cluster 2024-03-07T03:22:10.648595+0000 mon.a (mon.0) 972 : cluster 0 osdmap e647: 8 total, 7 up, 8 in 2024-03-07T03:22:12.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[79878]: audit 2024-03-07T03:22:10.662260+0000 mon.c (mon.2) 35 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2193661319,v1:172.21.15.145:6827/2193661319]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:22:12.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[79878]: audit 2024-03-07T03:22:10.662813+0000 mon.a (mon.0) 973 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:22:12.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[79878]: cluster 2024-03-07T03:22:10.716393+0000 mgr.y (mgr.34107) 302 : cluster 0 pgmap v582: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 9.6 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 921 KiB/s wr, 1 op/s; 70/519 objects degraded (13.487%) 2024-03-07T03:22:12.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[79878]: audit 2024-03-07T03:22:10.824978+0000 mon.a (mon.0) 974 : audit 0 from='client.? 172.21.15.145:0/1268657275' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:12.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[79878]: audit 2024-03-07T03:22:11.196262+0000 mon.a (mon.0) 975 : audit 1 from='osd.3 ' entity='osd.3' 2024-03-07T03:22:12.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[82329]: audit 2024-03-07T03:22:10.644862+0000 mon.a (mon.0) 971 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-07T03:22:12.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[82329]: cluster 2024-03-07T03:22:10.648595+0000 mon.a (mon.0) 972 : cluster 0 osdmap e647: 8 total, 7 up, 8 in 2024-03-07T03:22:12.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[82329]: audit 2024-03-07T03:22:10.662260+0000 mon.c (mon.2) 35 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2193661319,v1:172.21.15.145:6827/2193661319]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:22:12.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[82329]: audit 2024-03-07T03:22:10.662813+0000 mon.a (mon.0) 973 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:22:12.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[82329]: cluster 2024-03-07T03:22:10.716393+0000 mgr.y (mgr.34107) 302 : cluster 0 pgmap v582: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 9.6 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 921 KiB/s wr, 1 op/s; 70/519 objects degraded (13.487%) 2024-03-07T03:22:12.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[82329]: audit 2024-03-07T03:22:10.824978+0000 mon.a (mon.0) 974 : audit 0 from='client.? 172.21.15.145:0/1268657275' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:12.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:11 smithi145 bash[82329]: audit 2024-03-07T03:22:11.196262+0000 mon.a (mon.0) 975 : audit 1 from='osd.3 ' entity='osd.3' 2024-03-07T03:22:12.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:11 smithi189 bash[66969]: audit 2024-03-07T03:22:10.644862+0000 mon.a (mon.0) 971 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-07T03:22:12.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:11 smithi189 bash[66969]: cluster 2024-03-07T03:22:10.648595+0000 mon.a (mon.0) 972 : cluster 0 osdmap e647: 8 total, 7 up, 8 in 2024-03-07T03:22:12.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:11 smithi189 bash[66969]: audit 2024-03-07T03:22:10.662260+0000 mon.c (mon.2) 35 : audit 1 from='osd.3 [v2:172.21.15.145:6826/2193661319,v1:172.21.15.145:6827/2193661319]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:22:12.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:11 smithi189 bash[66969]: audit 2024-03-07T03:22:10.662813+0000 mon.a (mon.0) 973 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi145", "root=default"]}]: dispatch 2024-03-07T03:22:12.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:11 smithi189 bash[66969]: cluster 2024-03-07T03:22:10.716393+0000 mgr.y (mgr.34107) 302 : cluster 0 pgmap v582: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 9.6 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 921 KiB/s wr, 1 op/s; 70/519 objects degraded (13.487%) 2024-03-07T03:22:12.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:11 smithi189 bash[66969]: audit 2024-03-07T03:22:10.824978+0000 mon.a (mon.0) 974 : audit 0 from='client.? 172.21.15.145:0/1268657275' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:12.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:11 smithi189 bash[66969]: audit 2024-03-07T03:22:11.196262+0000 mon.a (mon.0) 975 : audit 1 from='osd.3 ' entity='osd.3' 2024-03-07T03:22:12.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:12 smithi145 bash[79878]: debug 2024-03-07T03:22:12.193+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:22:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:22:12 smithi189 bash[59814]: ts=2024-03-07T03:22:12.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:22:13.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:22:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:22:12] "GET /metrics HTTP/1.1" 200 60519 "" "Prometheus/2.43.0" 2024-03-07T03:22:13.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:12 smithi145 bash[79878]: cluster 2024-03-07T03:22:12.196095+0000 mon.a (mon.0) 976 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:22:13.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:12 smithi145 bash[79878]: cluster 2024-03-07T03:22:12.196169+0000 mon.a (mon.0) 977 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:22:13.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:12 smithi145 bash[79878]: cluster 2024-03-07T03:22:12.205856+0000 mon.a (mon.0) 978 : cluster 1 osd.3 [v2:172.21.15.145:6826/2193661319,v1:172.21.15.145:6827/2193661319] boot 2024-03-07T03:22:13.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:12 smithi145 bash[79878]: cluster 2024-03-07T03:22:12.205951+0000 mon.a (mon.0) 979 : cluster 0 osdmap e648: 8 total, 8 up, 8 in 2024-03-07T03:22:13.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:12 smithi145 bash[79878]: audit 2024-03-07T03:22:12.207041+0000 mon.a (mon.0) 980 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:22:13.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:12 smithi145 bash[82329]: cluster 2024-03-07T03:22:12.196095+0000 mon.a (mon.0) 976 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:22:13.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:12 smithi145 bash[82329]: cluster 2024-03-07T03:22:12.196169+0000 mon.a (mon.0) 977 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:22:13.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:12 smithi145 bash[82329]: cluster 2024-03-07T03:22:12.205856+0000 mon.a (mon.0) 978 : cluster 1 osd.3 [v2:172.21.15.145:6826/2193661319,v1:172.21.15.145:6827/2193661319] boot 2024-03-07T03:22:13.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:12 smithi145 bash[82329]: cluster 2024-03-07T03:22:12.205951+0000 mon.a (mon.0) 979 : cluster 0 osdmap e648: 8 total, 8 up, 8 in 2024-03-07T03:22:13.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:12 smithi145 bash[82329]: audit 2024-03-07T03:22:12.207041+0000 mon.a (mon.0) 980 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:22:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:12 smithi189 bash[66969]: cluster 2024-03-07T03:22:12.196095+0000 mon.a (mon.0) 976 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:22:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:12 smithi189 bash[66969]: cluster 2024-03-07T03:22:12.196169+0000 mon.a (mon.0) 977 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:22:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:12 smithi189 bash[66969]: cluster 2024-03-07T03:22:12.205856+0000 mon.a (mon.0) 978 : cluster 1 osd.3 [v2:172.21.15.145:6826/2193661319,v1:172.21.15.145:6827/2193661319] boot 2024-03-07T03:22:13.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:12 smithi189 bash[66969]: cluster 2024-03-07T03:22:12.205951+0000 mon.a (mon.0) 979 : cluster 0 osdmap e648: 8 total, 8 up, 8 in 2024-03-07T03:22:13.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:12 smithi189 bash[66969]: audit 2024-03-07T03:22:12.207041+0000 mon.a (mon.0) 980 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:22:13.184 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 4 2024-03-07T03:22:13.820 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:13 smithi145 bash[79878]: cluster 2024-03-07T03:22:12.717844+0000 mgr.y (mgr.34107) 303 : cluster 0 pgmap v584: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 9.6 MiB data, 462 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 640 KiB/s wr, 2 op/s; 70/519 objects degraded (13.487%) 2024-03-07T03:22:13.820 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:13 smithi145 bash[79878]: audit 2024-03-07T03:22:13.181315+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.145:0/4011831049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:13.820 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:13 smithi145 bash[79878]: cluster 2024-03-07T03:22:13.205301+0000 mon.a (mon.0) 982 : cluster 0 osdmap e649: 8 total, 8 up, 8 in 2024-03-07T03:22:13.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:13 smithi145 bash[82329]: cluster 2024-03-07T03:22:12.717844+0000 mgr.y (mgr.34107) 303 : cluster 0 pgmap v584: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 9.6 MiB data, 462 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 640 KiB/s wr, 2 op/s; 70/519 objects degraded (13.487%) 2024-03-07T03:22:13.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:13 smithi145 bash[82329]: audit 2024-03-07T03:22:13.181315+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.145:0/4011831049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:13.821 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:13 smithi145 bash[82329]: cluster 2024-03-07T03:22:13.205301+0000 mon.a (mon.0) 982 : cluster 0 osdmap e649: 8 total, 8 up, 8 in 2024-03-07T03:22:14.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:13 smithi189 bash[66969]: cluster 2024-03-07T03:22:12.717844+0000 mgr.y (mgr.34107) 303 : cluster 0 pgmap v584: 425 pgs: 46 active+undersized+degraded, 135 active+undersized, 244 active+clean; 9.6 MiB data, 462 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 640 KiB/s wr, 2 op/s; 70/519 objects degraded (13.487%) 2024-03-07T03:22:14.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:13 smithi189 bash[66969]: audit 2024-03-07T03:22:13.181315+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.145:0/4011831049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:14.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:13 smithi189 bash[66969]: cluster 2024-03-07T03:22:13.205301+0000 mon.a (mon.0) 982 : cluster 0 osdmap e649: 8 total, 8 up, 8 in 2024-03-07T03:22:14.478 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade pause' 2024-03-07T03:22:15.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:14 smithi189 bash[66969]: cluster 2024-03-07T03:22:14.774029+0000 mon.a (mon.0) 983 : cluster 3 Health check update: Degraded data redundancy: 42/519 objects degraded (8.092%), 31 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:15.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:14 smithi145 bash[82329]: cluster 2024-03-07T03:22:14.774029+0000 mon.a (mon.0) 983 : cluster 3 Health check update: Degraded data redundancy: 42/519 objects degraded (8.092%), 31 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:15.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:14 smithi145 bash[79878]: cluster 2024-03-07T03:22:14.774029+0000 mon.a (mon.0) 983 : cluster 3 Health check update: Degraded data redundancy: 42/519 objects degraded (8.092%), 31 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:16.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:15 smithi189 bash[66969]: cluster 2024-03-07T03:22:14.720698+0000 mgr.y (mgr.34107) 304 : cluster 0 pgmap v586: 425 pgs: 31 active+undersized+degraded, 83 active+undersized, 311 active+clean; 13 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 512 KiB/s wr, 1 op/s; 42/519 objects degraded (8.092%); 0 B/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:22:16.213 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:15 smithi145 bash[79878]: cluster 2024-03-07T03:22:14.720698+0000 mgr.y (mgr.34107) 304 : cluster 0 pgmap v586: 425 pgs: 31 active+undersized+degraded, 83 active+undersized, 311 active+clean; 13 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 512 KiB/s wr, 1 op/s; 42/519 objects degraded (8.092%); 0 B/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:22:16.214 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:15 smithi145 bash[82329]: cluster 2024-03-07T03:22:14.720698+0000 mgr.y (mgr.34107) 304 : cluster 0 pgmap v586: 425 pgs: 31 active+undersized+degraded, 83 active+undersized, 311 active+clean; 13 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 512 KiB/s wr, 1 op/s; 42/519 objects degraded (8.092%); 0 B/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:22:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:17 smithi145 bash[79878]: cluster 2024-03-07T03:22:16.722927+0000 mgr.y (mgr.34107) 305 : cluster 0 pgmap v587: 425 pgs: 4 active+undersized+degraded, 13 active+undersized, 408 active+clean; 17 MiB data, 479 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.2 MiB/s wr, 3 op/s; 5/522 objects degraded (0.958%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:22:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:17 smithi145 bash[79878]: cluster 2024-03-07T03:22:17.373982+0000 mon.a (mon.0) 984 : cluster 3 pool 'test-librbd-smithi145-68337-25' is full (reached quota's max_bytes: 10 MiB) 2024-03-07T03:22:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:17 smithi145 bash[79878]: cluster 2024-03-07T03:22:17.374311+0000 mon.a (mon.0) 985 : cluster 3 Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-07T03:22:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:17 smithi145 bash[79878]: cluster 2024-03-07T03:22:17.386420+0000 mon.a (mon.0) 986 : cluster 0 osdmap e650: 8 total, 8 up, 8 in 2024-03-07T03:22:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:17 smithi145 bash[79878]: audit 2024-03-07T03:22:17.650803+0000 mon.a (mon.0) 987 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:17 smithi145 bash[79878]: audit 2024-03-07T03:22:17.653743+0000 mon.a (mon.0) 988 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:18.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:17 smithi145 bash[82329]: cluster 2024-03-07T03:22:16.722927+0000 mgr.y (mgr.34107) 305 : cluster 0 pgmap v587: 425 pgs: 4 active+undersized+degraded, 13 active+undersized, 408 active+clean; 17 MiB data, 479 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.2 MiB/s wr, 3 op/s; 5/522 objects degraded (0.958%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:22:18.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:17 smithi145 bash[82329]: cluster 2024-03-07T03:22:17.373982+0000 mon.a (mon.0) 984 : cluster 3 pool 'test-librbd-smithi145-68337-25' is full (reached quota's max_bytes: 10 MiB) 2024-03-07T03:22:18.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:17 smithi145 bash[82329]: cluster 2024-03-07T03:22:17.374311+0000 mon.a (mon.0) 985 : cluster 3 Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-07T03:22:18.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:17 smithi145 bash[82329]: cluster 2024-03-07T03:22:17.386420+0000 mon.a (mon.0) 986 : cluster 0 osdmap e650: 8 total, 8 up, 8 in 2024-03-07T03:22:18.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:17 smithi145 bash[82329]: audit 2024-03-07T03:22:17.650803+0000 mon.a (mon.0) 987 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:18.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:17 smithi145 bash[82329]: audit 2024-03-07T03:22:17.653743+0000 mon.a (mon.0) 988 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:18.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:17 smithi189 bash[66969]: cluster 2024-03-07T03:22:16.722927+0000 mgr.y (mgr.34107) 305 : cluster 0 pgmap v587: 425 pgs: 4 active+undersized+degraded, 13 active+undersized, 408 active+clean; 17 MiB data, 479 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.2 MiB/s wr, 3 op/s; 5/522 objects degraded (0.958%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:22:18.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:17 smithi189 bash[66969]: cluster 2024-03-07T03:22:17.373982+0000 mon.a (mon.0) 984 : cluster 3 pool 'test-librbd-smithi145-68337-25' is full (reached quota's max_bytes: 10 MiB) 2024-03-07T03:22:18.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:17 smithi189 bash[66969]: cluster 2024-03-07T03:22:17.374311+0000 mon.a (mon.0) 985 : cluster 3 Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-07T03:22:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:17 smithi189 bash[66969]: cluster 2024-03-07T03:22:17.386420+0000 mon.a (mon.0) 986 : cluster 0 osdmap e650: 8 total, 8 up, 8 in 2024-03-07T03:22:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:17 smithi189 bash[66969]: audit 2024-03-07T03:22:17.650803+0000 mon.a (mon.0) 987 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:17 smithi189 bash[66969]: audit 2024-03-07T03:22:17.653743+0000 mon.a (mon.0) 988 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:18.486 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.RemoveFullTryDataPool (25705 ms) 2024-03-07T03:22:18.486 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestIO 2024-03-07T03:22:18.486 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:22:19.079 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:22:18 smithi189 bash[49584]: logger=sqlstore.transactions t=2024-03-07T03:22:18.740886099Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-07T03:22:19.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:18 smithi189 bash[66969]: cluster 2024-03-07T03:22:18.724182+0000 mgr.y (mgr.34107) 306 : cluster 0 pgmap v589: 425 pgs: 425 active+clean; 4.6 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 1.1 MiB/s wr, 33 op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:22:19.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:18 smithi189 bash[66969]: cluster 2024-03-07T03:22:18.836126+0000 mon.a (mon.0) 989 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/522 objects degraded (0.958%), 4 pgs degraded) 2024-03-07T03:22:19.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:18 smithi145 bash[79878]: cluster 2024-03-07T03:22:18.724182+0000 mgr.y (mgr.34107) 306 : cluster 0 pgmap v589: 425 pgs: 425 active+clean; 4.6 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 1.1 MiB/s wr, 33 op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:22:19.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:18 smithi145 bash[79878]: cluster 2024-03-07T03:22:18.836126+0000 mon.a (mon.0) 989 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/522 objects degraded (0.958%), 4 pgs degraded) 2024-03-07T03:22:19.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:18 smithi145 bash[82329]: cluster 2024-03-07T03:22:18.724182+0000 mgr.y (mgr.34107) 306 : cluster 0 pgmap v589: 425 pgs: 425 active+clean; 4.6 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 1.1 MiB/s wr, 33 op/s; 5 B/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:22:19.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:18 smithi145 bash[82329]: cluster 2024-03-07T03:22:18.836126+0000 mon.a (mon.0) 989 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/522 objects degraded (0.958%), 4 pgs degraded) 2024-03-07T03:22:20.224 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:22:21.274 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[79878]: audit 2024-03-07T03:22:19.906760+0000 mon.a (mon.0) 990 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:21.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[79878]: audit 2024-03-07T03:22:19.915869+0000 mon.a (mon.0) 991 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:21.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[79878]: audit 2024-03-07T03:22:19.917675+0000 mon.a (mon.0) 992 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:21.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[79878]: audit 2024-03-07T03:22:19.919217+0000 mon.a (mon.0) 993 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:21.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[79878]: audit 2024-03-07T03:22:19.932683+0000 mon.a (mon.0) 994 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:21.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[79878]: audit 2024-03-07T03:22:20.031096+0000 mon.a (mon.0) 995 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:21.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[79878]: audit 2024-03-07T03:22:20.035489+0000 mon.a (mon.0) 996 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:21.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[79878]: audit 2024-03-07T03:22:20.039101+0000 mon.a (mon.0) 997 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:21.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[79878]: audit 2024-03-07T03:22:20.041725+0000 mon.a (mon.0) 998 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:21.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[79878]: audit 2024-03-07T03:22:20.045024+0000 mon.a (mon.0) 999 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-07T03:22:21.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[79878]: audit 2024-03-07T03:22:20.045233+0000 mgr.y (mgr.34107) 307 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-07T03:22:21.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[79878]: cephadm 2024-03-07T03:22:20.047385+0000 mgr.y (mgr.34107) 308 : cephadm 1 Upgrade: osd.4 is safe to restart 2024-03-07T03:22:21.275 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[79878]: cluster 2024-03-07T03:22:20.725182+0000 mgr.y (mgr.34107) 309 : cluster 0 pgmap v590: 425 pgs: 425 active+clean; 4.6 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 901 KiB/s wr, 27 op/s; 4 B/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:22:21.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[82329]: audit 2024-03-07T03:22:19.906760+0000 mon.a (mon.0) 990 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:21.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[82329]: audit 2024-03-07T03:22:19.915869+0000 mon.a (mon.0) 991 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:21.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[82329]: audit 2024-03-07T03:22:19.917675+0000 mon.a (mon.0) 992 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:21.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[82329]: audit 2024-03-07T03:22:19.919217+0000 mon.a (mon.0) 993 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:21.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[82329]: audit 2024-03-07T03:22:19.932683+0000 mon.a (mon.0) 994 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:21.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[82329]: audit 2024-03-07T03:22:20.031096+0000 mon.a (mon.0) 995 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:21.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[82329]: audit 2024-03-07T03:22:20.035489+0000 mon.a (mon.0) 996 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:21.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[82329]: audit 2024-03-07T03:22:20.039101+0000 mon.a (mon.0) 997 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:21.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[82329]: audit 2024-03-07T03:22:20.041725+0000 mon.a (mon.0) 998 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:21.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[82329]: audit 2024-03-07T03:22:20.045024+0000 mon.a (mon.0) 999 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-07T03:22:21.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[82329]: audit 2024-03-07T03:22:20.045233+0000 mgr.y (mgr.34107) 307 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-07T03:22:21.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[82329]: cephadm 2024-03-07T03:22:20.047385+0000 mgr.y (mgr.34107) 308 : cephadm 1 Upgrade: osd.4 is safe to restart 2024-03-07T03:22:21.276 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:20 smithi145 bash[82329]: cluster 2024-03-07T03:22:20.725182+0000 mgr.y (mgr.34107) 309 : cluster 0 pgmap v590: 425 pgs: 425 active+clean; 4.6 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 901 KiB/s wr, 27 op/s; 4 B/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:22:21.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:20 smithi189 bash[66969]: audit 2024-03-07T03:22:19.906760+0000 mon.a (mon.0) 990 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:20 smithi189 bash[66969]: audit 2024-03-07T03:22:19.915869+0000 mon.a (mon.0) 991 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:20 smithi189 bash[66969]: audit 2024-03-07T03:22:19.917675+0000 mon.a (mon.0) 992 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:20 smithi189 bash[66969]: audit 2024-03-07T03:22:19.919217+0000 mon.a (mon.0) 993 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:20 smithi189 bash[66969]: audit 2024-03-07T03:22:19.932683+0000 mon.a (mon.0) 994 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:20 smithi189 bash[66969]: audit 2024-03-07T03:22:20.031096+0000 mon.a (mon.0) 995 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:20 smithi189 bash[66969]: audit 2024-03-07T03:22:20.035489+0000 mon.a (mon.0) 996 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:20 smithi189 bash[66969]: audit 2024-03-07T03:22:20.039101+0000 mon.a (mon.0) 997 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:20 smithi189 bash[66969]: audit 2024-03-07T03:22:20.041725+0000 mon.a (mon.0) 998 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:20 smithi189 bash[66969]: audit 2024-03-07T03:22:20.045024+0000 mon.a (mon.0) 999 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-07T03:22:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:20 smithi189 bash[66969]: audit 2024-03-07T03:22:20.045233+0000 mgr.y (mgr.34107) 307 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-07T03:22:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:20 smithi189 bash[66969]: cephadm 2024-03-07T03:22:20.047385+0000 mgr.y (mgr.34107) 308 : cephadm 1 Upgrade: osd.4 is safe to restart 2024-03-07T03:22:21.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:20 smithi189 bash[66969]: cluster 2024-03-07T03:22:20.725182+0000 mgr.y (mgr.34107) 309 : cluster 0 pgmap v590: 425 pgs: 425 active+clean; 4.6 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 901 KiB/s wr, 27 op/s; 4 B/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:22:22.417 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:22:22 smithi189 bash[59814]: ts=2024-03-07T03:22:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:22:22.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:22 smithi145 bash[79878]: cluster 2024-03-07T03:22:22.375773+0000 mon.a (mon.0) 1000 : cluster 1 pool 'test-librbd-smithi145-68337-25' no longer out of quota; removing NO_QUOTA flag 2024-03-07T03:22:22.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:22 smithi145 bash[79878]: cluster 2024-03-07T03:22:22.376133+0000 mon.a (mon.0) 1001 : cluster 1 Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-07T03:22:22.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:22 smithi145 bash[79878]: cluster 2024-03-07T03:22:22.384039+0000 mon.a (mon.0) 1002 : cluster 0 osdmap e651: 8 total, 8 up, 8 in 2024-03-07T03:22:22.750 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:22 smithi145 bash[82329]: cluster 2024-03-07T03:22:22.375773+0000 mon.a (mon.0) 1000 : cluster 1 pool 'test-librbd-smithi145-68337-25' no longer out of quota; removing NO_QUOTA flag 2024-03-07T03:22:22.750 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:22 smithi145 bash[82329]: cluster 2024-03-07T03:22:22.376133+0000 mon.a (mon.0) 1001 : cluster 1 Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-07T03:22:22.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:22 smithi145 bash[82329]: cluster 2024-03-07T03:22:22.384039+0000 mon.a (mon.0) 1002 : cluster 0 osdmap e651: 8 total, 8 up, 8 in 2024-03-07T03:22:22.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:22 smithi189 bash[66969]: cluster 2024-03-07T03:22:22.375773+0000 mon.a (mon.0) 1000 : cluster 1 pool 'test-librbd-smithi145-68337-25' no longer out of quota; removing NO_QUOTA flag 2024-03-07T03:22:22.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:22 smithi189 bash[66969]: cluster 2024-03-07T03:22:22.376133+0000 mon.a (mon.0) 1001 : cluster 1 Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-07T03:22:22.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:22 smithi189 bash[66969]: cluster 2024-03-07T03:22:22.384039+0000 mon.a (mon.0) 1002 : cluster 0 osdmap e651: 8 total, 8 up, 8 in 2024-03-07T03:22:22.969 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:22.969 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:22.969 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:22.969 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:22.969 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.970 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:22.971 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.972 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:22.973 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:22.974 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 512~512 = 512 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:aio discard: 1536~512 = 0 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:22.975 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 0 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 0 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 16384 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 16384 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:22.976 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 49152 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 49152 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 65536 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 65536 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 98304 2024-03-07T03:22:22.977 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 98304 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 114688 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 114688 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 147456 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.978 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 147456 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 163840 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 163840 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 196608 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 196608 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.979 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 212992 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 212992 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.980 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.981 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.982 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:22.983 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.983 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.983 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:22.983 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:22.983 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:22.983 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:22.983 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:22.983 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:22.983 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:22.983 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:23.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:22:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:22:22] "GET /metrics HTTP/1.1" 200 60638 "" "Prometheus/2.43.0" 2024-03-07T03:22:23.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:23 smithi145 bash[82329]: cephadm 2024-03-07T03:22:22.666693+0000 mgr.y (mgr.34107) 310 : cephadm 1 Upgrade: Updating osd.4 2024-03-07T03:22:23.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:23 smithi145 bash[82329]: audit 2024-03-07T03:22:22.672925+0000 mon.a (mon.0) 1003 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:23.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:23 smithi145 bash[82329]: audit 2024-03-07T03:22:22.676093+0000 mon.a (mon.0) 1004 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:22:23.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:23 smithi145 bash[82329]: audit 2024-03-07T03:22:22.677407+0000 mon.a (mon.0) 1005 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:23.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:23 smithi145 bash[82329]: cephadm 2024-03-07T03:22:22.688303+0000 mgr.y (mgr.34107) 311 : cephadm 1 Deploying daemon osd.4 on smithi189 2024-03-07T03:22:23.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:23 smithi145 bash[82329]: cluster 2024-03-07T03:22:22.726513+0000 mgr.y (mgr.34107) 312 : cluster 0 pgmap v592: 425 pgs: 425 active+clean; 601 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 646 KiB/s wr, 37 op/s; 4 B/s, 0 objects/s recovering 2024-03-07T03:22:23.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:23 smithi145 bash[79878]: cephadm 2024-03-07T03:22:22.666693+0000 mgr.y (mgr.34107) 310 : cephadm 1 Upgrade: Updating osd.4 2024-03-07T03:22:23.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:23 smithi145 bash[79878]: audit 2024-03-07T03:22:22.672925+0000 mon.a (mon.0) 1003 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:23.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:23 smithi145 bash[79878]: audit 2024-03-07T03:22:22.676093+0000 mon.a (mon.0) 1004 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:22:23.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:23 smithi145 bash[79878]: audit 2024-03-07T03:22:22.677407+0000 mon.a (mon.0) 1005 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:23.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:23 smithi145 bash[79878]: cephadm 2024-03-07T03:22:22.688303+0000 mgr.y (mgr.34107) 311 : cephadm 1 Deploying daemon osd.4 on smithi189 2024-03-07T03:22:23.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:23 smithi145 bash[79878]: cluster 2024-03-07T03:22:22.726513+0000 mgr.y (mgr.34107) 312 : cluster 0 pgmap v592: 425 pgs: 425 active+clean; 601 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 646 KiB/s wr, 37 op/s; 4 B/s, 0 objects/s recovering 2024-03-07T03:22:23.947 INFO:teuthology.orchestra.run.smithi145.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:22:24.020 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:23 smithi189 bash[66969]: cephadm 2024-03-07T03:22:22.666693+0000 mgr.y (mgr.34107) 310 : cephadm 1 Upgrade: Updating osd.4 2024-03-07T03:22:24.020 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:23 smithi189 bash[66969]: audit 2024-03-07T03:22:22.672925+0000 mon.a (mon.0) 1003 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:24.020 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:23 smithi189 bash[66969]: audit 2024-03-07T03:22:22.676093+0000 mon.a (mon.0) 1004 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:22:24.021 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:23 smithi189 bash[66969]: audit 2024-03-07T03:22:22.677407+0000 mon.a (mon.0) 1005 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:24.021 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:23 smithi189 bash[66969]: cephadm 2024-03-07T03:22:22.688303+0000 mgr.y (mgr.34107) 311 : cephadm 1 Deploying daemon osd.4 on smithi189 2024-03-07T03:22:24.021 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:23 smithi189 bash[66969]: cluster 2024-03-07T03:22:22.726513+0000 mgr.y (mgr.34107) 312 : cluster 0 pgmap v592: 425 pgs: 425 active+clean; 601 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 646 KiB/s wr, 37 op/s; 4 B/s, 0 objects/s recovering 2024-03-07T03:22:24.847 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch ps' 2024-03-07T03:22:25.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:24 smithi145 bash[79878]: audit 2024-03-07T03:22:23.767014+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:25.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:24 smithi145 bash[79878]: audit 2024-03-07T03:22:23.959266+0000 mon.a (mon.0) 1007 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:25.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:24 smithi145 bash[82329]: audit 2024-03-07T03:22:23.767014+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:25.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:24 smithi145 bash[82329]: audit 2024-03-07T03:22:23.959266+0000 mon.a (mon.0) 1007 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:25.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:24 smithi189 bash[66969]: audit 2024-03-07T03:22:23.767014+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:25.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:24 smithi189 bash[66969]: audit 2024-03-07T03:22:23.959266+0000 mon.a (mon.0) 1007 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:25.948 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.949 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.949 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.949 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.949 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.949 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.949 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.949 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.949 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.949 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.949 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.949 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.949 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.950 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.950 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.950 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:25.950 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.950 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.950 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:25.950 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.950 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.950 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.950 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.950 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.951 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.951 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.951 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.951 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.951 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.951 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.951 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.951 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.951 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.951 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.951 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.951 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.952 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.952 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.952 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.952 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.952 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.952 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.952 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.952 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.952 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.952 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.952 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.952 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.952 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.953 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:25.953 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.953 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.953 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:25.953 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.953 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.953 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.953 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.953 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.953 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.953 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.953 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.953 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.954 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.954 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.954 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.954 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.954 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.954 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.954 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.954 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.954 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.954 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.954 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.954 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.955 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.955 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.955 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.955 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.955 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.955 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.955 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.955 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.955 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.955 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:25.955 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.955 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.955 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:25.956 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.956 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.956 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.956 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.956 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.956 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.956 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.956 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.956 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.956 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.956 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.956 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.957 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.957 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.957 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.957 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.957 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.957 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.957 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.957 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.957 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.957 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.957 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.957 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:25.958 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:25.958 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:25.958 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:25.958 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:25.958 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.958 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.958 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:25.958 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:25.958 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:25.958 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:25.958 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:25.958 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:25.959 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:25.959 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:25.959 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestIO (7462 ms) 2024-03-07T03:22:25.959 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS1 2024-03-07T03:22:25.959 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:22:26.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:25 smithi145 bash[79878]: audit 2024-03-07T03:22:23.954991+0000 mgr.y (mgr.34107) 313 : audit 0 from='client.44525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:22:26.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:25 smithi145 bash[79878]: cephadm 2024-03-07T03:22:23.955327+0000 mgr.y (mgr.34107) 314 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:22:26.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:25 smithi145 bash[79878]: cluster 2024-03-07T03:22:24.727949+0000 mgr.y (mgr.34107) 315 : cluster 0 pgmap v593: 425 pgs: 425 active+clean; 697 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 349 KiB/s rd, 304 KiB/s wr, 679 op/s 2024-03-07T03:22:26.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:25 smithi145 bash[82329]: audit 2024-03-07T03:22:23.954991+0000 mgr.y (mgr.34107) 313 : audit 0 from='client.44525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:22:26.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:25 smithi145 bash[82329]: cephadm 2024-03-07T03:22:23.955327+0000 mgr.y (mgr.34107) 314 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:22:26.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:25 smithi145 bash[82329]: cluster 2024-03-07T03:22:24.727949+0000 mgr.y (mgr.34107) 315 : cluster 0 pgmap v593: 425 pgs: 425 active+clean; 697 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 349 KiB/s rd, 304 KiB/s wr, 679 op/s 2024-03-07T03:22:26.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:25 smithi189 bash[66969]: audit 2024-03-07T03:22:23.954991+0000 mgr.y (mgr.34107) 313 : audit 0 from='client.44525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:22:26.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:25 smithi189 bash[66969]: cephadm 2024-03-07T03:22:23.955327+0000 mgr.y (mgr.34107) 314 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:22:26.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:25 smithi189 bash[66969]: cluster 2024-03-07T03:22:24.727949+0000 mgr.y (mgr.34107) 315 : cluster 0 pgmap v593: 425 pgs: 425 active+clean; 697 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 349 KiB/s rd, 304 KiB/s wr, 679 op/s 2024-03-07T03:22:27.351 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:27.351 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:22:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:27.352 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:27.352 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:22:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:27.352 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:22:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:27.353 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:22:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:27.353 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:22:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:27.353 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:22:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:27.353 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:22:27 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:27.791 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:27 smithi189 systemd[1]: Stopping Ceph osd.4 for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:22:27.791 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:27 smithi189 bash[27162]: debug 2024-03-07T03:22:27.477+0000 7f34ffa28700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:22:27.791 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:27 smithi189 bash[27162]: debug 2024-03-07T03:22:27.477+0000 7f34ffa28700 -1 osd.4 651 *** Got signal Terminated *** 2024-03-07T03:22:27.791 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:27 smithi189 bash[27162]: debug 2024-03-07T03:22:27.477+0000 7f34ffa28700 -1 osd.4 651 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:22:28.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:27 smithi145 bash[79878]: debug 2024-03-07T03:22:27.768+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:22:28.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:27 smithi145 bash[79878]: cluster 2024-03-07T03:22:26.730032+0000 mgr.y (mgr.34107) 316 : cluster 0 pgmap v594: 425 pgs: 425 active+clean; 697 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 299 KiB/s rd, 260 KiB/s wr, 581 op/s 2024-03-07T03:22:28.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:27 smithi145 bash[79878]: cluster 2024-03-07T03:22:27.483528+0000 mon.a (mon.0) 1008 : cluster 1 osd.4 marked itself down and dead 2024-03-07T03:22:28.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:27 smithi145 bash[82329]: cluster 2024-03-07T03:22:26.730032+0000 mgr.y (mgr.34107) 316 : cluster 0 pgmap v594: 425 pgs: 425 active+clean; 697 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 299 KiB/s rd, 260 KiB/s wr, 581 op/s 2024-03-07T03:22:28.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:27 smithi145 bash[82329]: cluster 2024-03-07T03:22:27.483528+0000 mon.a (mon.0) 1008 : cluster 1 osd.4 marked itself down and dead 2024-03-07T03:22:28.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:27 smithi189 bash[66969]: cluster 2024-03-07T03:22:26.730032+0000 mgr.y (mgr.34107) 316 : cluster 0 pgmap v594: 425 pgs: 425 active+clean; 697 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 299 KiB/s rd, 260 KiB/s wr, 581 op/s 2024-03-07T03:22:28.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:27 smithi189 bash[66969]: cluster 2024-03-07T03:22:27.483528+0000 mon.a (mon.0) 1008 : cluster 1 osd.4 marked itself down and dead 2024-03-07T03:22:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:28 smithi145 bash[79878]: cluster 2024-03-07T03:22:27.771184+0000 mon.a (mon.0) 1009 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:22:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:28 smithi145 bash[79878]: cluster 2024-03-07T03:22:27.771235+0000 mon.a (mon.0) 1010 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:22:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:28 smithi145 bash[79878]: cluster 2024-03-07T03:22:27.781299+0000 mon.a (mon.0) 1011 : cluster 0 osdmap e652: 8 total, 7 up, 8 in 2024-03-07T03:22:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:28 smithi145 bash[79878]: cluster 2024-03-07T03:22:27.787495+0000 mon.a (mon.0) 1012 : cluster 1 osd.4 marked itself dead as of e652 2024-03-07T03:22:29.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:28 smithi145 bash[79878]: cluster 2024-03-07T03:22:28.785506+0000 mon.a (mon.0) 1013 : cluster 0 osdmap e653: 8 total, 7 up, 8 in 2024-03-07T03:22:29.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:28 smithi145 bash[82329]: cluster 2024-03-07T03:22:27.771184+0000 mon.a (mon.0) 1009 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:22:29.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:28 smithi145 bash[82329]: cluster 2024-03-07T03:22:27.771235+0000 mon.a (mon.0) 1010 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:22:29.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:28 smithi145 bash[82329]: cluster 2024-03-07T03:22:27.781299+0000 mon.a (mon.0) 1011 : cluster 0 osdmap e652: 8 total, 7 up, 8 in 2024-03-07T03:22:29.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:28 smithi145 bash[82329]: cluster 2024-03-07T03:22:27.787495+0000 mon.a (mon.0) 1012 : cluster 1 osd.4 marked itself dead as of e652 2024-03-07T03:22:29.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:28 smithi145 bash[82329]: cluster 2024-03-07T03:22:28.785506+0000 mon.a (mon.0) 1013 : cluster 0 osdmap e653: 8 total, 7 up, 8 in 2024-03-07T03:22:29.079 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:29 smithi189 bash[72084]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-4 2024-03-07T03:22:29.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:28 smithi189 bash[66969]: cluster 2024-03-07T03:22:27.771184+0000 mon.a (mon.0) 1009 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:22:29.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:28 smithi189 bash[66969]: cluster 2024-03-07T03:22:27.771235+0000 mon.a (mon.0) 1010 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:22:29.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:28 smithi189 bash[66969]: cluster 2024-03-07T03:22:27.781299+0000 mon.a (mon.0) 1011 : cluster 0 osdmap e652: 8 total, 7 up, 8 in 2024-03-07T03:22:29.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:28 smithi189 bash[66969]: cluster 2024-03-07T03:22:27.787495+0000 mon.a (mon.0) 1012 : cluster 1 osd.4 marked itself dead as of e652 2024-03-07T03:22:29.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:28 smithi189 bash[66969]: cluster 2024-03-07T03:22:28.785506+0000 mon.a (mon.0) 1013 : cluster 0 osdmap e653: 8 total, 7 up, 8 in 2024-03-07T03:22:29.553 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:22:29.579 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:29 smithi189 bash[72150]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-4 2024-03-07T03:22:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:29 smithi145 bash[79878]: cluster 2024-03-07T03:22:28.732426+0000 mgr.y (mgr.34107) 317 : cluster 0 pgmap v596: 425 pgs: 68 peering, 31 stale+active+clean, 326 active+clean; 2.6 MiB data, 426 MiB used, 715 GiB / 715 GiB avail; 514 KiB/s rd, 387 KiB/s wr, 1.04k op/s 2024-03-07T03:22:30.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:29 smithi145 bash[79878]: cluster 2024-03-07T03:22:28.799922+0000 mon.a (mon.0) 1014 : cluster 3 Health check failed: Reduced data availability: 7 pgs inactive, 13 pgs peering (PG_AVAILABILITY) 2024-03-07T03:22:30.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:29 smithi145 bash[82329]: cluster 2024-03-07T03:22:28.732426+0000 mgr.y (mgr.34107) 317 : cluster 0 pgmap v596: 425 pgs: 68 peering, 31 stale+active+clean, 326 active+clean; 2.6 MiB data, 426 MiB used, 715 GiB / 715 GiB avail; 514 KiB/s rd, 387 KiB/s wr, 1.04k op/s 2024-03-07T03:22:30.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:29 smithi145 bash[82329]: cluster 2024-03-07T03:22:28.799922+0000 mon.a (mon.0) 1014 : cluster 3 Health check failed: Reduced data availability: 7 pgs inactive, 13 pgs peering (PG_AVAILABILITY) 2024-03-07T03:22:30.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:29 smithi189 bash[66969]: cluster 2024-03-07T03:22:28.732426+0000 mgr.y (mgr.34107) 317 : cluster 0 pgmap v596: 425 pgs: 68 peering, 31 stale+active+clean, 326 active+clean; 2.6 MiB data, 426 MiB used, 715 GiB / 715 GiB avail; 514 KiB/s rd, 387 KiB/s wr, 1.04k op/s 2024-03-07T03:22:30.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:29 smithi189 bash[66969]: cluster 2024-03-07T03:22:28.799922+0000 mon.a (mon.0) 1014 : cluster 3 Health check failed: Reduced data availability: 7 pgs inactive, 13 pgs peering (PG_AVAILABILITY) 2024-03-07T03:22:31.644 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:22:31 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:31.644 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:22:31 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:31.644 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:31 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:31.645 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:22:31 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:31.645 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:22:31 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:31.645 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:22:31 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:31.645 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:22:31 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:31.646 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:31 smithi189 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.4.service: Deactivated successfully. 2024-03-07T03:22:31.646 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:31 smithi189 systemd[1]: Stopped Ceph osd.4 for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:22:31.646 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:31 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:31.646 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:22:31 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:22:32.083 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:31 smithi189 systemd[1]: Started Ceph osd.4 for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:22:32.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:31 smithi189 bash[66969]: cluster 2024-03-07T03:22:30.733600+0000 mgr.y (mgr.34107) 318 : cluster 0 pgmap v598: 425 pgs: 68 peering, 31 stale+active+clean, 326 active+clean; 2.6 MiB data, 426 MiB used, 715 GiB / 715 GiB avail; 508 KiB/s rd, 258 KiB/s wr, 1.03k op/s 2024-03-07T03:22:32.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:31 smithi189 bash[66969]: audit 2024-03-07T03:22:31.707472+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:32.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:31 smithi189 bash[66969]: audit 2024-03-07T03:22:31.721102+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:32.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:31 smithi189 bash[66969]: audit 2024-03-07T03:22:31.770504+0000 mon.a (mon.0) 1017 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:32.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:31 smithi145 bash[82329]: cluster 2024-03-07T03:22:30.733600+0000 mgr.y (mgr.34107) 318 : cluster 0 pgmap v598: 425 pgs: 68 peering, 31 stale+active+clean, 326 active+clean; 2.6 MiB data, 426 MiB used, 715 GiB / 715 GiB avail; 508 KiB/s rd, 258 KiB/s wr, 1.03k op/s 2024-03-07T03:22:32.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:31 smithi145 bash[82329]: audit 2024-03-07T03:22:31.707472+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:32.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:31 smithi145 bash[82329]: audit 2024-03-07T03:22:31.721102+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:32.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:31 smithi145 bash[82329]: audit 2024-03-07T03:22:31.770504+0000 mon.a (mon.0) 1017 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:32.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:31 smithi145 bash[79878]: cluster 2024-03-07T03:22:30.733600+0000 mgr.y (mgr.34107) 318 : cluster 0 pgmap v598: 425 pgs: 68 peering, 31 stale+active+clean, 326 active+clean; 2.6 MiB data, 426 MiB used, 715 GiB / 715 GiB avail; 508 KiB/s rd, 258 KiB/s wr, 1.03k op/s 2024-03-07T03:22:32.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:31 smithi145 bash[79878]: audit 2024-03-07T03:22:31.707472+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:32.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:31 smithi145 bash[79878]: audit 2024-03-07T03:22:31.721102+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:32.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:31 smithi145 bash[79878]: audit 2024-03-07T03:22:31.770504+0000 mon.a (mon.0) 1017 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:22:32 smithi189 bash[59814]: ts=2024-03-07T03:22:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:22:33.035 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:22:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:22:32] "GET /metrics HTTP/1.1" 200 60608 "" "Prometheus/2.43.0" 2024-03-07T03:22:33.477 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:33.477 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:33.477 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:33.477 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:33.477 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:33.477 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.477 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:33.477 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.477 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.477 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:33.477 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.477 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:33.478 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:33.479 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.480 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:33.481 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 512~512 = 512 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:aio discard: 1536~512 = 0 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:33.483 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 0 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 0 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 16384 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 16384 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:33.484 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 49152 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 49152 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 65536 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 65536 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 98304 2024-03-07T03:22:33.485 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 98304 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 114688 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 114688 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 147456 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.486 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 147456 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 163840 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 163840 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 196608 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.487 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 196608 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 212992 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 212992 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:33.488 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.490 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.491 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:33.491 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.491 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.491 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.491 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:33.491 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:33.491 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:33.491 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:33.491 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:33.491 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:33.491 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:33.492 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-07T03:22:33.492 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (15m) 25s ago 16m 15.0M - 0.25.0 c8568f914cd2 cfa1e64a6007 2024-03-07T03:22:33.492 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi189 *:3000 running (15m) 54s ago 15m 78.8M - 9.4.7 954c08fa6188 89badf11bd11 2024-03-07T03:22:33.492 INFO:teuthology.orchestra.run.smithi145.stdout:mds.foo.smithi145.zmgvgw smithi145 running (13m) 25s ago 13m 15.8M - 18.2.1-718-gae75fc9d 0164bb8a7c67 4ded79fef80e 2024-03-07T03:22:33.492 INFO:teuthology.orchestra.run.smithi145.stdout:mds.foo.smithi189.zkinbe smithi189 running (13m) 54s ago 13m 20.0M - 18.2.1-718-gae75fc9d 0164bb8a7c67 6589deb5c181 2024-03-07T03:22:33.492 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi189 *:8443,9283,8765 running (8m) 54s ago 24m 439M - 19.0.0-1905-gea9a5688 fdb83767c1ab acef232ce9b9 2024-03-07T03:22:33.492 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (8m) 25s ago 26m 507M - 19.0.0-1905-gea9a5688 fdb83767c1ab 2cae6fe389e0 2024-03-07T03:22:33.492 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (7m) 25s ago 26m 66.5M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 8a2f4984bf09 2024-03-07T03:22:33.492 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi189 running (5m) 54s ago 24m 45.1M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab c2bb73c80f4c 2024-03-07T03:22:33.492 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (5m) 25s ago 24m 51.9M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 1c9cf0604d03 2024-03-07T03:22:33.493 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (16m) 25s ago 16m 9516k - 1.5.0 0da6a335fe13 895e660f39b6 2024-03-07T03:22:33.493 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi189 *:9100 running (16m) 54s ago 16m 9071k - 1.5.0 0da6a335fe13 ab7ffb60c1e8 2024-03-07T03:22:33.493 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (3m) 25s ago 23m 83.3M 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab b5501c15a04b 2024-03-07T03:22:33.493 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (3m) 25s ago 22m 85.8M 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab bf7b3fb25071 2024-03-07T03:22:33.493 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (58s) 25s ago 21m 92.3M 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab 401d59281db3 2024-03-07T03:22:33.493 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (28s) 25s ago 20m - 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab 7c830bbb24f4 2024-03-07T03:22:33.493 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi189 starting - - - 2524M 2024-03-07T03:22:33.493 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi189 running (19m) 54s ago 19m 123M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 44e5411eb07c 2024-03-07T03:22:33.493 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi189 running (18m) 54s ago 18m 149M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 c2aeea0466c7 2024-03-07T03:22:33.493 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi189 running (17m) 54s ago 17m 123M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 bf101bfb6800 2024-03-07T03:22:33.493 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi189 *:9095 running (8m) 54s ago 16m 54.4M - 2.43.0 a07b618ecd1d 144e5cadeea3 2024-03-07T03:22:33.579 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:33 smithi189 bash[72299]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-07T03:22:33.580 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:33 smithi189 bash[72299]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-07T03:22:33.580 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:33 smithi189 bash[72299]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-07T03:22:33.580 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:33 smithi189 bash[72299]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-07T03:22:33.580 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:33 smithi189 bash[72299]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-03-07T03:22:33.580 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:33 smithi189 bash[72299]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-07T03:22:33.580 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:33 smithi189 bash[72299]: --> ceph-volume raw activate successful for osd ID: 4 2024-03-07T03:22:33.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:33 smithi145 bash[82329]: audit 2024-03-07T03:22:32.646741+0000 mon.a (mon.0) 1018 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:33.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:33 smithi145 bash[82329]: audit 2024-03-07T03:22:32.648527+0000 mon.a (mon.0) 1019 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:33.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:33 smithi145 bash[82329]: cluster 2024-03-07T03:22:32.735262+0000 mgr.y (mgr.34107) 319 : cluster 0 pgmap v599: 425 pgs: 25 active+undersized+degraded, 68 active+undersized, 68 peering, 264 active+clean; 2.6 MiB data, 439 MiB used, 715 GiB / 715 GiB avail; 178 KiB/s rd, 89 KiB/s wr, 385 op/s; 40/525 objects degraded (7.619%) 2024-03-07T03:22:33.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:33 smithi145 bash[79878]: audit 2024-03-07T03:22:32.646741+0000 mon.a (mon.0) 1018 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:33.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:33 smithi145 bash[79878]: audit 2024-03-07T03:22:32.648527+0000 mon.a (mon.0) 1019 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:33.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:33 smithi145 bash[79878]: cluster 2024-03-07T03:22:32.735262+0000 mgr.y (mgr.34107) 319 : cluster 0 pgmap v599: 425 pgs: 25 active+undersized+degraded, 68 active+undersized, 68 peering, 264 active+clean; 2.6 MiB data, 439 MiB used, 715 GiB / 715 GiB avail; 178 KiB/s rd, 89 KiB/s wr, 385 op/s; 40/525 objects degraded (7.619%) 2024-03-07T03:22:34.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:33 smithi189 bash[66969]: audit 2024-03-07T03:22:32.646741+0000 mon.a (mon.0) 1018 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:34.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:33 smithi189 bash[66969]: audit 2024-03-07T03:22:32.648527+0000 mon.a (mon.0) 1019 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:34.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:33 smithi189 bash[66969]: cluster 2024-03-07T03:22:32.735262+0000 mgr.y (mgr.34107) 319 : cluster 0 pgmap v599: 425 pgs: 25 active+undersized+degraded, 68 active+undersized, 68 peering, 264 active+clean; 2.6 MiB data, 439 MiB used, 715 GiB / 715 GiB avail; 178 KiB/s rd, 89 KiB/s wr, 385 op/s; 40/525 objects degraded (7.619%) 2024-03-07T03:22:34.376 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch ps' 2024-03-07T03:22:35.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:34 smithi145 bash[79878]: audit 2024-03-07T03:22:33.479157+0000 mgr.y (mgr.34107) 320 : audit 0 from='client.44531 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:22:35.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:34 smithi145 bash[79878]: cluster 2024-03-07T03:22:33.643981+0000 mon.a (mon.0) 1020 : cluster 3 Health check failed: Degraded data redundancy: 40/525 objects degraded (7.619%), 25 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:35.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:34 smithi145 bash[82329]: audit 2024-03-07T03:22:33.479157+0000 mgr.y (mgr.34107) 320 : audit 0 from='client.44531 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:22:35.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:34 smithi145 bash[82329]: cluster 2024-03-07T03:22:33.643981+0000 mon.a (mon.0) 1020 : cluster 3 Health check failed: Degraded data redundancy: 40/525 objects degraded (7.619%), 25 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:35.077 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:34 smithi189 bash[66969]: audit 2024-03-07T03:22:33.479157+0000 mgr.y (mgr.34107) 320 : audit 0 from='client.44531 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:22:35.077 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:34 smithi189 bash[66969]: cluster 2024-03-07T03:22:33.643981+0000 mon.a (mon.0) 1020 : cluster 3 Health check failed: Degraded data redundancy: 40/525 objects degraded (7.619%), 25 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:35.329 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:35 smithi189 bash[72532]: debug 2024-03-07T03:22:35.181+0000 7fd7f17f2700 1 -- 172.21.15.189:0/677618869 <== mon.1 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558f45ec5680 con 0x558f45f00800 2024-03-07T03:22:36.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:35 smithi145 bash[79878]: cluster 2024-03-07T03:22:34.737992+0000 mgr.y (mgr.34107) 321 : cluster 0 pgmap v600: 425 pgs: 43 active+undersized+degraded, 118 active+undersized, 264 active+clean; 6.8 MiB data, 444 MiB used, 715 GiB / 715 GiB avail; 758 KiB/s rd, 897 KiB/s wr, 971 op/s; 79/531 objects degraded (14.878%) 2024-03-07T03:22:36.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:35 smithi145 bash[82329]: cluster 2024-03-07T03:22:34.737992+0000 mgr.y (mgr.34107) 321 : cluster 0 pgmap v600: 425 pgs: 43 active+undersized+degraded, 118 active+undersized, 264 active+clean; 6.8 MiB data, 444 MiB used, 715 GiB / 715 GiB avail; 758 KiB/s rd, 897 KiB/s wr, 971 op/s; 79/531 objects degraded (14.878%) 2024-03-07T03:22:36.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:35 smithi189 bash[66969]: cluster 2024-03-07T03:22:34.737992+0000 mgr.y (mgr.34107) 321 : cluster 0 pgmap v600: 425 pgs: 43 active+undersized+degraded, 118 active+undersized, 264 active+clean; 6.8 MiB data, 444 MiB used, 715 GiB / 715 GiB avail; 758 KiB/s rd, 897 KiB/s wr, 971 op/s; 79/531 objects degraded (14.878%) 2024-03-07T03:22:36.079 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:35 smithi189 bash[72532]: debug 2024-03-07T03:22:35.729+0000 7fd7fa98a7c0 -1 Falling back to public interface 2024-03-07T03:22:37.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:36 smithi145 bash[79878]: cluster 2024-03-07T03:22:35.653367+0000 mon.a (mon.0) 1021 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 14 pgs peering) 2024-03-07T03:22:37.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:36 smithi145 bash[82329]: cluster 2024-03-07T03:22:35.653367+0000 mon.a (mon.0) 1021 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 14 pgs peering) 2024-03-07T03:22:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:36 smithi189 bash[66969]: cluster 2024-03-07T03:22:35.653367+0000 mon.a (mon.0) 1021 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 14 pgs peering) 2024-03-07T03:22:38.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:37 smithi145 bash[82329]: cluster 2024-03-07T03:22:36.740789+0000 mgr.y (mgr.34107) 322 : cluster 0 pgmap v601: 425 pgs: 43 active+undersized+degraded, 118 active+undersized, 264 active+clean; 6.8 MiB data, 445 MiB used, 715 GiB / 715 GiB avail; 678 KiB/s rd, 801 KiB/s wr, 868 op/s; 79/531 objects degraded (14.878%) 2024-03-07T03:22:38.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:37 smithi145 bash[79878]: cluster 2024-03-07T03:22:36.740789+0000 mgr.y (mgr.34107) 322 : cluster 0 pgmap v601: 425 pgs: 43 active+undersized+degraded, 118 active+undersized, 264 active+clean; 6.8 MiB data, 445 MiB used, 715 GiB / 715 GiB avail; 678 KiB/s rd, 801 KiB/s wr, 868 op/s; 79/531 objects degraded (14.878%) 2024-03-07T03:22:38.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:37 smithi189 bash[66969]: cluster 2024-03-07T03:22:36.740789+0000 mgr.y (mgr.34107) 322 : cluster 0 pgmap v601: 425 pgs: 43 active+undersized+degraded, 118 active+undersized, 264 active+clean; 6.8 MiB data, 445 MiB used, 715 GiB / 715 GiB avail; 678 KiB/s rd, 801 KiB/s wr, 868 op/s; 79/531 objects degraded (14.878%) 2024-03-07T03:22:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:38 smithi189 bash[66969]: audit 2024-03-07T03:22:37.824666+0000 mon.a (mon.0) 1022 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:38 smithi189 bash[66969]: audit 2024-03-07T03:22:37.835631+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:39.080 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:38 smithi189 bash[72532]: debug 2024-03-07T03:22:38.917+0000 7fd7fa98a7c0 -1 osd.4 652 log_to_monitors true 2024-03-07T03:22:39.104 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:22:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:38 smithi145 bash[79878]: audit 2024-03-07T03:22:37.824666+0000 mon.a (mon.0) 1022 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:39.123 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:38 smithi145 bash[79878]: audit 2024-03-07T03:22:37.835631+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:39.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:38 smithi145 bash[82329]: audit 2024-03-07T03:22:37.824666+0000 mon.a (mon.0) 1022 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:39.123 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:38 smithi145 bash[82329]: audit 2024-03-07T03:22:37.835631+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:40.079 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:22:39 smithi189 bash[72532]: debug 2024-03-07T03:22:39.889+0000 7fd7ebde2700 -1 osd.4 652 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-07T03:22:40.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:39 smithi189 bash[66969]: cluster 2024-03-07T03:22:38.743748+0000 mgr.y (mgr.34107) 323 : cluster 0 pgmap v602: 425 pgs: 43 active+undersized+degraded, 118 active+undersized, 264 active+clean; 6.8 MiB data, 444 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 830 KiB/s wr, 841 op/s; 79/531 objects degraded (14.878%) 2024-03-07T03:22:40.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:39 smithi189 bash[66969]: audit 2024-03-07T03:22:38.927850+0000 mon.b (mon.1) 3 : audit 1 from='osd.4 [v2:172.21.15.189:6800/553416210,v1:172.21.15.189:6801/553416210]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:22:40.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:39 smithi189 bash[66969]: audit 2024-03-07T03:22:38.928054+0000 mon.a (mon.0) 1024 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:22:40.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:39 smithi189 bash[66969]: audit 2024-03-07T03:22:38.936039+0000 mon.a (mon.0) 1025 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:40.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:39 smithi189 bash[66969]: audit 2024-03-07T03:22:38.945231+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:40.215 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:39 smithi145 bash[82329]: cluster 2024-03-07T03:22:38.743748+0000 mgr.y (mgr.34107) 323 : cluster 0 pgmap v602: 425 pgs: 43 active+undersized+degraded, 118 active+undersized, 264 active+clean; 6.8 MiB data, 444 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 830 KiB/s wr, 841 op/s; 79/531 objects degraded (14.878%) 2024-03-07T03:22:40.215 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:39 smithi145 bash[82329]: audit 2024-03-07T03:22:38.927850+0000 mon.b (mon.1) 3 : audit 1 from='osd.4 [v2:172.21.15.189:6800/553416210,v1:172.21.15.189:6801/553416210]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:22:40.215 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:39 smithi145 bash[82329]: audit 2024-03-07T03:22:38.928054+0000 mon.a (mon.0) 1024 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:22:40.215 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:39 smithi145 bash[82329]: audit 2024-03-07T03:22:38.936039+0000 mon.a (mon.0) 1025 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:40.215 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:39 smithi145 bash[82329]: audit 2024-03-07T03:22:38.945231+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:40.215 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:39 smithi145 bash[79878]: cluster 2024-03-07T03:22:38.743748+0000 mgr.y (mgr.34107) 323 : cluster 0 pgmap v602: 425 pgs: 43 active+undersized+degraded, 118 active+undersized, 264 active+clean; 6.8 MiB data, 444 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 830 KiB/s wr, 841 op/s; 79/531 objects degraded (14.878%) 2024-03-07T03:22:40.215 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:39 smithi145 bash[79878]: audit 2024-03-07T03:22:38.927850+0000 mon.b (mon.1) 3 : audit 1 from='osd.4 [v2:172.21.15.189:6800/553416210,v1:172.21.15.189:6801/553416210]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:22:40.215 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:39 smithi145 bash[79878]: audit 2024-03-07T03:22:38.928054+0000 mon.a (mon.0) 1024 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:22:40.215 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:39 smithi145 bash[79878]: audit 2024-03-07T03:22:38.936039+0000 mon.a (mon.0) 1025 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:40.215 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:39 smithi145 bash[79878]: audit 2024-03-07T03:22:38.945231+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:41.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:40 smithi145 bash[82329]: audit 2024-03-07T03:22:39.842768+0000 mon.a (mon.0) 1027 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-07T03:22:41.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:40 smithi145 bash[82329]: cluster 2024-03-07T03:22:39.845951+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e654: 8 total, 7 up, 8 in 2024-03-07T03:22:41.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:40 smithi145 bash[82329]: audit 2024-03-07T03:22:39.853686+0000 mon.b (mon.1) 4 : audit 1 from='osd.4 [v2:172.21.15.189:6800/553416210,v1:172.21.15.189:6801/553416210]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:22:41.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:40 smithi145 bash[82329]: audit 2024-03-07T03:22:39.853896+0000 mon.a (mon.0) 1029 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:22:41.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:40 smithi145 bash[79878]: debug 2024-03-07T03:22:40.840+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:22:41.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:40 smithi145 bash[79878]: audit 2024-03-07T03:22:39.842768+0000 mon.a (mon.0) 1027 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-07T03:22:41.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:40 smithi145 bash[79878]: cluster 2024-03-07T03:22:39.845951+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e654: 8 total, 7 up, 8 in 2024-03-07T03:22:41.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:40 smithi145 bash[79878]: audit 2024-03-07T03:22:39.853686+0000 mon.b (mon.1) 4 : audit 1 from='osd.4 [v2:172.21.15.189:6800/553416210,v1:172.21.15.189:6801/553416210]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:22:41.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:40 smithi145 bash[79878]: audit 2024-03-07T03:22:39.853896+0000 mon.a (mon.0) 1029 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:22:41.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:40 smithi189 bash[66969]: audit 2024-03-07T03:22:39.842768+0000 mon.a (mon.0) 1027 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-07T03:22:41.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:40 smithi189 bash[66969]: cluster 2024-03-07T03:22:39.845951+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e654: 8 total, 7 up, 8 in 2024-03-07T03:22:41.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:40 smithi189 bash[66969]: audit 2024-03-07T03:22:39.853686+0000 mon.b (mon.1) 4 : audit 1 from='osd.4 [v2:172.21.15.189:6800/553416210,v1:172.21.15.189:6801/553416210]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:22:41.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:40 smithi189 bash[66969]: audit 2024-03-07T03:22:39.853896+0000 mon.a (mon.0) 1029 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-03-07T03:22:41.915 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.915 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.915 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.915 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.915 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.915 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.915 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.915 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.916 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.917 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.918 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.919 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:41.920 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.921 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.922 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 4 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:read: 4 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:read: 4 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS1 (15967 ms) 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS2 2024-03-07T03:22:41.923 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:22:42.191 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[79878]: cluster 2024-03-07T03:22:40.745620+0000 mgr.y (mgr.34107) 324 : cluster 0 pgmap v604: 425 pgs: 43 active+undersized+degraded, 118 active+undersized, 264 active+clean; 6.8 MiB data, 415 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 830 KiB/s wr, 841 op/s; 79/531 objects degraded (14.878%) 2024-03-07T03:22:42.191 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[79878]: cluster 2024-03-07T03:22:40.844560+0000 mon.a (mon.0) 1030 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:22:42.191 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[79878]: cluster 2024-03-07T03:22:40.844660+0000 mon.a (mon.0) 1031 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:22:42.191 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[79878]: cluster 2024-03-07T03:22:40.850559+0000 mon.a (mon.0) 1032 : cluster 1 osd.4 [v2:172.21.15.189:6800/553416210,v1:172.21.15.189:6801/553416210] boot 2024-03-07T03:22:42.191 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[79878]: cluster 2024-03-07T03:22:40.850601+0000 mon.a (mon.0) 1033 : cluster 0 osdmap e655: 8 total, 8 up, 8 in 2024-03-07T03:22:42.191 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[79878]: audit 2024-03-07T03:22:40.851163+0000 mon.a (mon.0) 1034 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:22:42.191 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[79878]: cluster 2024-03-07T03:22:41.854494+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e656: 8 total, 8 up, 8 in 2024-03-07T03:22:42.191 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[82329]: cluster 2024-03-07T03:22:40.745620+0000 mgr.y (mgr.34107) 324 : cluster 0 pgmap v604: 425 pgs: 43 active+undersized+degraded, 118 active+undersized, 264 active+clean; 6.8 MiB data, 415 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 830 KiB/s wr, 841 op/s; 79/531 objects degraded (14.878%) 2024-03-07T03:22:42.191 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[82329]: cluster 2024-03-07T03:22:40.844560+0000 mon.a (mon.0) 1030 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:22:42.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[82329]: cluster 2024-03-07T03:22:40.844660+0000 mon.a (mon.0) 1031 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:22:42.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[82329]: cluster 2024-03-07T03:22:40.850559+0000 mon.a (mon.0) 1032 : cluster 1 osd.4 [v2:172.21.15.189:6800/553416210,v1:172.21.15.189:6801/553416210] boot 2024-03-07T03:22:42.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[82329]: cluster 2024-03-07T03:22:40.850601+0000 mon.a (mon.0) 1033 : cluster 0 osdmap e655: 8 total, 8 up, 8 in 2024-03-07T03:22:42.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[82329]: audit 2024-03-07T03:22:40.851163+0000 mon.a (mon.0) 1034 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:22:42.192 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:41 smithi145 bash[82329]: cluster 2024-03-07T03:22:41.854494+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e656: 8 total, 8 up, 8 in 2024-03-07T03:22:42.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:41 smithi189 bash[66969]: cluster 2024-03-07T03:22:40.745620+0000 mgr.y (mgr.34107) 324 : cluster 0 pgmap v604: 425 pgs: 43 active+undersized+degraded, 118 active+undersized, 264 active+clean; 6.8 MiB data, 415 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 830 KiB/s wr, 841 op/s; 79/531 objects degraded (14.878%) 2024-03-07T03:22:42.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:41 smithi189 bash[66969]: cluster 2024-03-07T03:22:40.844560+0000 mon.a (mon.0) 1030 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:22:42.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:41 smithi189 bash[66969]: cluster 2024-03-07T03:22:40.844660+0000 mon.a (mon.0) 1031 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:22:42.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:41 smithi189 bash[66969]: cluster 2024-03-07T03:22:40.850559+0000 mon.a (mon.0) 1032 : cluster 1 osd.4 [v2:172.21.15.189:6800/553416210,v1:172.21.15.189:6801/553416210] boot 2024-03-07T03:22:42.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:41 smithi189 bash[66969]: cluster 2024-03-07T03:22:40.850601+0000 mon.a (mon.0) 1033 : cluster 0 osdmap e655: 8 total, 8 up, 8 in 2024-03-07T03:22:42.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:41 smithi189 bash[66969]: audit 2024-03-07T03:22:40.851163+0000 mon.a (mon.0) 1034 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:22:42.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:41 smithi189 bash[66969]: cluster 2024-03-07T03:22:41.854494+0000 mon.a (mon.0) 1035 : cluster 0 osdmap e656: 8 total, 8 up, 8 in 2024-03-07T03:22:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:22:42 smithi189 bash[59814]: ts=2024-03-07T03:22:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:22:42.580 INFO:teuthology.orchestra.run.smithi145.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-07T03:22:42.580 INFO:teuthology.orchestra.run.smithi145.stdout:alertmanager.a smithi145 *:9093,9094 running (15m) 34s ago 16m 15.0M - 0.25.0 c8568f914cd2 cfa1e64a6007 2024-03-07T03:22:42.580 INFO:teuthology.orchestra.run.smithi145.stdout:grafana.a smithi189 *:3000 running (15m) 4s ago 15m 79.8M - 9.4.7 954c08fa6188 89badf11bd11 2024-03-07T03:22:42.580 INFO:teuthology.orchestra.run.smithi145.stdout:mds.foo.smithi145.zmgvgw smithi145 running (13m) 34s ago 13m 15.8M - 18.2.1-718-gae75fc9d 0164bb8a7c67 4ded79fef80e 2024-03-07T03:22:42.580 INFO:teuthology.orchestra.run.smithi145.stdout:mds.foo.smithi189.zkinbe smithi189 running (13m) 4s ago 13m 20.4M - 18.2.1-718-gae75fc9d 0164bb8a7c67 6589deb5c181 2024-03-07T03:22:42.580 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.x smithi189 *:8443,9283,8765 running (8m) 4s ago 24m 439M - 19.0.0-1905-gea9a5688 fdb83767c1ab acef232ce9b9 2024-03-07T03:22:42.580 INFO:teuthology.orchestra.run.smithi145.stdout:mgr.y smithi145 *:8443,9283,8765 running (9m) 34s ago 26m 507M - 19.0.0-1905-gea9a5688 fdb83767c1ab 2cae6fe389e0 2024-03-07T03:22:42.580 INFO:teuthology.orchestra.run.smithi145.stdout:mon.a smithi145 running (7m) 34s ago 27m 66.5M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 8a2f4984bf09 2024-03-07T03:22:42.580 INFO:teuthology.orchestra.run.smithi145.stdout:mon.b smithi189 running (5m) 4s ago 24m 49.5M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab c2bb73c80f4c 2024-03-07T03:22:42.581 INFO:teuthology.orchestra.run.smithi145.stdout:mon.c smithi145 running (6m) 34s ago 24m 51.9M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 1c9cf0604d03 2024-03-07T03:22:42.581 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.a smithi145 *:9100 running (16m) 34s ago 16m 9516k - 1.5.0 0da6a335fe13 895e660f39b6 2024-03-07T03:22:42.581 INFO:teuthology.orchestra.run.smithi145.stdout:node-exporter.b smithi189 *:9100 running (16m) 4s ago 16m 9564k - 1.5.0 0da6a335fe13 ab7ffb60c1e8 2024-03-07T03:22:42.581 INFO:teuthology.orchestra.run.smithi145.stdout:osd.0 smithi145 running (3m) 34s ago 23m 83.3M 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab b5501c15a04b 2024-03-07T03:22:42.581 INFO:teuthology.orchestra.run.smithi145.stdout:osd.1 smithi145 running (3m) 34s ago 22m 85.8M 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab bf7b3fb25071 2024-03-07T03:22:42.581 INFO:teuthology.orchestra.run.smithi145.stdout:osd.2 smithi145 running (67s) 34s ago 21m 92.3M 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab 401d59281db3 2024-03-07T03:22:42.581 INFO:teuthology.orchestra.run.smithi145.stdout:osd.3 smithi145 running (37s) 34s ago 20m - 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab 7c830bbb24f4 2024-03-07T03:22:42.581 INFO:teuthology.orchestra.run.smithi145.stdout:osd.4 smithi189 running (8s) 4s ago 20m - 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab b805c0ededd2 2024-03-07T03:22:42.581 INFO:teuthology.orchestra.run.smithi145.stdout:osd.5 smithi189 running (19m) 4s ago 19m 127M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 44e5411eb07c 2024-03-07T03:22:42.581 INFO:teuthology.orchestra.run.smithi145.stdout:osd.6 smithi189 running (18m) 4s ago 18m 151M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 c2aeea0466c7 2024-03-07T03:22:42.581 INFO:teuthology.orchestra.run.smithi145.stdout:osd.7 smithi189 running (17m) 4s ago 17m 135M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 bf101bfb6800 2024-03-07T03:22:42.581 INFO:teuthology.orchestra.run.smithi145.stdout:prometheus.a smithi189 *:9095 running (8m) 4s ago 16m 57.0M - 2.43.0 a07b618ecd1d 144e5cadeea3 2024-03-07T03:22:42.862 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:22:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:22:42] "GET /metrics HTTP/1.1" 200 60608 "" "Prometheus/2.43.0" 2024-03-07T03:22:43.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:42 smithi145 bash[79878]: cluster 2024-03-07T03:22:42.380216+0000 mon.a (mon.0) 1036 : cluster 3 Health check update: Degraded data redundancy: 79/531 objects degraded (14.878%), 43 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:43.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:42 smithi145 bash[82329]: cluster 2024-03-07T03:22:42.380216+0000 mon.a (mon.0) 1036 : cluster 3 Health check update: Degraded data redundancy: 79/531 objects degraded (14.878%), 43 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:43.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:42 smithi189 bash[66969]: cluster 2024-03-07T03:22:42.380216+0000 mon.a (mon.0) 1036 : cluster 3 Health check update: Degraded data redundancy: 79/531 objects degraded (14.878%), 43 pgs degraded (PG_DEGRADED) 2024-03-07T03:22:43.797 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph versions' 2024-03-07T03:22:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:43 smithi145 bash[79878]: audit 2024-03-07T03:22:42.578435+0000 mgr.y (mgr.34107) 325 : audit 0 from='client.34623 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:22:44.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:43 smithi145 bash[79878]: cluster 2024-03-07T03:22:42.747087+0000 mgr.y (mgr.34107) 326 : cluster 0 pgmap v607: 425 pgs: 34 peering, 33 active+undersized+degraded, 94 active+undersized, 264 active+clean; 11 MiB data, 414 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 305 KiB/s wr, 634 op/s; 66/534 objects degraded (12.360%) 2024-03-07T03:22:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:43 smithi145 bash[82329]: audit 2024-03-07T03:22:42.578435+0000 mgr.y (mgr.34107) 325 : audit 0 from='client.34623 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:22:44.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:43 smithi145 bash[82329]: cluster 2024-03-07T03:22:42.747087+0000 mgr.y (mgr.34107) 326 : cluster 0 pgmap v607: 425 pgs: 34 peering, 33 active+undersized+degraded, 94 active+undersized, 264 active+clean; 11 MiB data, 414 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 305 KiB/s wr, 634 op/s; 66/534 objects degraded (12.360%) 2024-03-07T03:22:44.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:43 smithi189 bash[66969]: audit 2024-03-07T03:22:42.578435+0000 mgr.y (mgr.34107) 325 : audit 0 from='client.34623 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:22:44.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:43 smithi189 bash[66969]: cluster 2024-03-07T03:22:42.747087+0000 mgr.y (mgr.34107) 326 : cluster 0 pgmap v607: 425 pgs: 34 peering, 33 active+undersized+degraded, 94 active+undersized, 264 active+clean; 11 MiB data, 414 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 305 KiB/s wr, 634 op/s; 66/534 objects degraded (12.360%) 2024-03-07T03:22:46.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:45 smithi189 bash[66969]: cluster 2024-03-07T03:22:44.748716+0000 mgr.y (mgr.34107) 327 : cluster 0 pgmap v608: 425 pgs: 34 peering, 21 active+undersized+degraded, 60 active+undersized, 310 active+clean; 11 MiB data, 398 MiB used, 715 GiB / 715 GiB avail; 739 KiB/s rd, 511 B/s wr, 12 op/s; 41/537 objects degraded (7.635%); 717 KiB/s, 0 objects/s recovering 2024-03-07T03:22:46.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:45 smithi145 bash[79878]: cluster 2024-03-07T03:22:44.748716+0000 mgr.y (mgr.34107) 327 : cluster 0 pgmap v608: 425 pgs: 34 peering, 21 active+undersized+degraded, 60 active+undersized, 310 active+clean; 11 MiB data, 398 MiB used, 715 GiB / 715 GiB avail; 739 KiB/s rd, 511 B/s wr, 12 op/s; 41/537 objects degraded (7.635%); 717 KiB/s, 0 objects/s recovering 2024-03-07T03:22:46.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:45 smithi145 bash[82329]: cluster 2024-03-07T03:22:44.748716+0000 mgr.y (mgr.34107) 327 : cluster 0 pgmap v608: 425 pgs: 34 peering, 21 active+undersized+degraded, 60 active+undersized, 310 active+clean; 11 MiB data, 398 MiB used, 715 GiB / 715 GiB avail; 739 KiB/s rd, 511 B/s wr, 12 op/s; 41/537 objects degraded (7.635%); 717 KiB/s, 0 objects/s recovering 2024-03-07T03:22:47.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:46 smithi145 bash[79878]: cluster 2024-03-07T03:22:46.750361+0000 mgr.y (mgr.34107) 328 : cluster 0 pgmap v609: 425 pgs: 18 peering, 407 active+clean; 11 MiB data, 403 MiB used, 715 GiB / 715 GiB avail; 647 KiB/s rd, 1.0 KiB/s wr, 16 op/s; 623 KiB/s, 0 objects/s recovering 2024-03-07T03:22:47.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:46 smithi145 bash[79878]: cluster 2024-03-07T03:22:46.885321+0000 mon.a (mon.0) 1037 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/537 objects degraded (7.635%), 21 pgs degraded) 2024-03-07T03:22:47.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:46 smithi145 bash[82329]: cluster 2024-03-07T03:22:46.750361+0000 mgr.y (mgr.34107) 328 : cluster 0 pgmap v609: 425 pgs: 18 peering, 407 active+clean; 11 MiB data, 403 MiB used, 715 GiB / 715 GiB avail; 647 KiB/s rd, 1.0 KiB/s wr, 16 op/s; 623 KiB/s, 0 objects/s recovering 2024-03-07T03:22:47.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:46 smithi145 bash[82329]: cluster 2024-03-07T03:22:46.885321+0000 mon.a (mon.0) 1037 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/537 objects degraded (7.635%), 21 pgs degraded) 2024-03-07T03:22:47.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:46 smithi189 bash[66969]: cluster 2024-03-07T03:22:46.750361+0000 mgr.y (mgr.34107) 328 : cluster 0 pgmap v609: 425 pgs: 18 peering, 407 active+clean; 11 MiB data, 403 MiB used, 715 GiB / 715 GiB avail; 647 KiB/s rd, 1.0 KiB/s wr, 16 op/s; 623 KiB/s, 0 objects/s recovering 2024-03-07T03:22:47.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:46 smithi189 bash[66969]: cluster 2024-03-07T03:22:46.885321+0000 mon.a (mon.0) 1037 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/537 objects degraded (7.635%), 21 pgs degraded) 2024-03-07T03:22:48.521 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:22:48.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:48 smithi145 bash[79878]: audit 2024-03-07T03:22:47.646933+0000 mon.a (mon.0) 1038 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:48.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:48 smithi145 bash[79878]: audit 2024-03-07T03:22:47.648830+0000 mon.a (mon.0) 1039 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:48.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:48 smithi145 bash[82329]: audit 2024-03-07T03:22:47.646933+0000 mon.a (mon.0) 1038 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:48.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:48 smithi145 bash[82329]: audit 2024-03-07T03:22:47.648830+0000 mon.a (mon.0) 1039 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:48 smithi189 bash[66969]: audit 2024-03-07T03:22:47.646933+0000 mon.a (mon.0) 1038 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:49.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:48 smithi189 bash[66969]: audit 2024-03-07T03:22:47.648830+0000 mon.a (mon.0) 1039 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:50.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[82329]: cluster 2024-03-07T03:22:48.751909+0000 mgr.y (mgr.34107) 329 : cluster 0 pgmap v610: 425 pgs: 425 active+clean; 11 MiB data, 403 MiB used, 715 GiB / 715 GiB avail; 558 KiB/s rd, 895 B/s wr, 13 op/s; 538 KiB/s, 0 objects/s recovering 2024-03-07T03:22:50.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[82329]: audit 2024-03-07T03:22:48.855911+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:50.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[82329]: audit 2024-03-07T03:22:48.863896+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:50.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[82329]: audit 2024-03-07T03:22:48.865500+0000 mon.a (mon.0) 1042 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:50.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[82329]: audit 2024-03-07T03:22:48.866585+0000 mon.a (mon.0) 1043 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:50.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[82329]: audit 2024-03-07T03:22:48.873971+0000 mon.a (mon.0) 1044 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:50.008 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[82329]: audit 2024-03-07T03:22:48.950232+0000 mon.a (mon.0) 1045 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:50.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[82329]: audit 2024-03-07T03:22:48.952228+0000 mon.a (mon.0) 1046 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:50.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[82329]: audit 2024-03-07T03:22:48.953201+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:50.009 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[82329]: audit 2024-03-07T03:22:48.959557+0000 mon.a (mon.0) 1048 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:50.009 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[79878]: cluster 2024-03-07T03:22:48.751909+0000 mgr.y (mgr.34107) 329 : cluster 0 pgmap v610: 425 pgs: 425 active+clean; 11 MiB data, 403 MiB used, 715 GiB / 715 GiB avail; 558 KiB/s rd, 895 B/s wr, 13 op/s; 538 KiB/s, 0 objects/s recovering 2024-03-07T03:22:50.009 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[79878]: audit 2024-03-07T03:22:48.855911+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:50.009 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[79878]: audit 2024-03-07T03:22:48.863896+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:50.009 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[79878]: audit 2024-03-07T03:22:48.865500+0000 mon.a (mon.0) 1042 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:50.009 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[79878]: audit 2024-03-07T03:22:48.866585+0000 mon.a (mon.0) 1043 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:50.009 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[79878]: audit 2024-03-07T03:22:48.873971+0000 mon.a (mon.0) 1044 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:50.009 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[79878]: audit 2024-03-07T03:22:48.950232+0000 mon.a (mon.0) 1045 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:50.009 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[79878]: audit 2024-03-07T03:22:48.952228+0000 mon.a (mon.0) 1046 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:50.009 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[79878]: audit 2024-03-07T03:22:48.953201+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:50.010 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:49 smithi145 bash[79878]: audit 2024-03-07T03:22:48.959557+0000 mon.a (mon.0) 1048 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:49 smithi189 bash[66969]: cluster 2024-03-07T03:22:48.751909+0000 mgr.y (mgr.34107) 329 : cluster 0 pgmap v610: 425 pgs: 425 active+clean; 11 MiB data, 403 MiB used, 715 GiB / 715 GiB avail; 558 KiB/s rd, 895 B/s wr, 13 op/s; 538 KiB/s, 0 objects/s recovering 2024-03-07T03:22:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:49 smithi189 bash[66969]: audit 2024-03-07T03:22:48.855911+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:49 smithi189 bash[66969]: audit 2024-03-07T03:22:48.863896+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:49 smithi189 bash[66969]: audit 2024-03-07T03:22:48.865500+0000 mon.a (mon.0) 1042 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:49 smithi189 bash[66969]: audit 2024-03-07T03:22:48.866585+0000 mon.a (mon.0) 1043 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:49 smithi189 bash[66969]: audit 2024-03-07T03:22:48.873971+0000 mon.a (mon.0) 1044 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:49 smithi189 bash[66969]: audit 2024-03-07T03:22:48.950232+0000 mon.a (mon.0) 1045 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:49 smithi189 bash[66969]: audit 2024-03-07T03:22:48.952228+0000 mon.a (mon.0) 1046 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:49 smithi189 bash[66969]: audit 2024-03-07T03:22:48.953201+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:50.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:49 smithi189 bash[66969]: audit 2024-03-07T03:22:48.959557+0000 mon.a (mon.0) 1048 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:52.035 INFO:teuthology.orchestra.run.smithi145.stdout:{ 2024-03-07T03:22:52.035 INFO:teuthology.orchestra.run.smithi145.stdout: "mon": { 2024-03-07T03:22:52.035 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 3 2024-03-07T03:22:52.035 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: "mgr": { 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 2 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: "osd": { 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable)": 3, 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 5 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: "mds": { 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable)": 2 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: }, 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: "overall": { 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable)": 5, 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 10 2024-03-07T03:22:52.036 INFO:teuthology.orchestra.run.smithi145.stdout: } 2024-03-07T03:22:52.037 INFO:teuthology.orchestra.run.smithi145.stdout:} 2024-03-07T03:22:52.290 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:51 smithi145 bash[79878]: cluster 2024-03-07T03:22:50.753760+0000 mgr.y (mgr.34107) 330 : cluster 0 pgmap v611: 425 pgs: 425 active+clean; 19 MiB data, 415 MiB used, 715 GiB / 715 GiB avail; 446 KiB/s rd, 828 KiB/s wr, 6 op/s; 435 KiB/s, 0 objects/s recovering 2024-03-07T03:22:52.291 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:51 smithi145 bash[82329]: cluster 2024-03-07T03:22:50.753760+0000 mgr.y (mgr.34107) 330 : cluster 0 pgmap v611: 425 pgs: 425 active+clean; 19 MiB data, 415 MiB used, 715 GiB / 715 GiB avail; 446 KiB/s rd, 828 KiB/s wr, 6 op/s; 435 KiB/s, 0 objects/s recovering 2024-03-07T03:22:52.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:51 smithi189 bash[66969]: cluster 2024-03-07T03:22:50.753760+0000 mgr.y (mgr.34107) 330 : cluster 0 pgmap v611: 425 pgs: 425 active+clean; 19 MiB data, 415 MiB used, 715 GiB / 715 GiB avail; 446 KiB/s rd, 828 KiB/s wr, 6 op/s; 435 KiB/s, 0 objects/s recovering 2024-03-07T03:22:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:22:52 smithi189 bash[59814]: ts=2024-03-07T03:22:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:22:52.895 DEBUG:teuthology.parallel:result is None 2024-03-07T03:22:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:52 smithi145 bash[79878]: audit 2024-03-07T03:22:52.046892+0000 mon.a (mon.0) 1049 : audit 0 from='client.? 172.21.15.145:0/3647881080' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:53.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:22:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:22:52] "GET /metrics HTTP/1.1" 200 60658 "" "Prometheus/2.43.0" 2024-03-07T03:22:53.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:52 smithi145 bash[82329]: audit 2024-03-07T03:22:52.046892+0000 mon.a (mon.0) 1049 : audit 0 from='client.? 172.21.15.145:0/3647881080' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:53.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:52 smithi189 bash[66969]: audit 2024-03-07T03:22:52.046892+0000 mon.a (mon.0) 1049 : audit 0 from='client.? 172.21.15.145:0/3647881080' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:54.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:53 smithi145 bash[79878]: cluster 2024-03-07T03:22:52.756117+0000 mgr.y (mgr.34107) 331 : cluster 0 pgmap v612: 425 pgs: 425 active+clean; 19 MiB data, 431 MiB used, 715 GiB / 715 GiB avail; 408 KiB/s rd, 753 KiB/s wr, 9 op/s; 395 KiB/s, 0 objects/s recovering 2024-03-07T03:22:54.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:53 smithi145 bash[79878]: audit 2024-03-07T03:22:53.816490+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:54.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:53 smithi145 bash[82329]: cluster 2024-03-07T03:22:52.756117+0000 mgr.y (mgr.34107) 331 : cluster 0 pgmap v612: 425 pgs: 425 active+clean; 19 MiB data, 431 MiB used, 715 GiB / 715 GiB avail; 408 KiB/s rd, 753 KiB/s wr, 9 op/s; 395 KiB/s, 0 objects/s recovering 2024-03-07T03:22:54.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:53 smithi145 bash[82329]: audit 2024-03-07T03:22:53.816490+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:54.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:53 smithi189 bash[66969]: cluster 2024-03-07T03:22:52.756117+0000 mgr.y (mgr.34107) 331 : cluster 0 pgmap v612: 425 pgs: 425 active+clean; 19 MiB data, 431 MiB used, 715 GiB / 715 GiB avail; 408 KiB/s rd, 753 KiB/s wr, 9 op/s; 395 KiB/s, 0 objects/s recovering 2024-03-07T03:22:54.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:53 smithi189 bash[66969]: audit 2024-03-07T03:22:53.816490+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:22:54.981 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:54.981 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:54.981 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:54.981 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:54.982 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:22:54.982 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.982 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:54.982 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.982 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:54.982 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:54.982 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.982 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:54.982 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.982 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:54.982 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:54.982 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.983 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:54.983 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.983 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:54.983 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:54.983 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.983 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:54.983 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.983 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:54.983 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:54.983 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.983 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:22:54.983 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.984 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:54.984 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:22:54.984 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:54.984 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:54.984 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:54.984 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:54.984 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:54.984 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:54.984 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:54.984 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:54.984 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:22:54.984 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:22:54.985 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.985 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:54.987 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.987 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:54.987 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:54.987 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.987 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:54.987 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.987 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:54.988 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:54.988 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.988 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:54.988 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.988 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:54.988 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:54.988 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.988 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:54.988 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.988 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:54.988 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:54.989 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.989 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:22:54.989 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.989 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:54.989 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:22:54.989 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:54.989 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:54.989 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:54.989 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:54.989 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:54.989 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.989 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:54.990 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:54.990 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:54.990 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.990 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:54.990 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:54.990 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:54.990 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.990 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:54.990 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:54.990 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:54.990 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.990 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:54.991 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:54.991 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:54.991 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.991 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:22:54.991 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:22:54.991 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:22:54.991 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 512~512 = 512 2024-03-07T03:22:54.991 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.991 INFO:tasks.workunit.client.0.smithi145.stdout:aio discard: 1536~512 = 0 2024-03-07T03:22:54.991 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:54.991 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:54.991 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:54.992 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:54.992 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:22:54.992 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 0 2024-03-07T03:22:54.992 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.992 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.992 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 0 2024-03-07T03:22:54.992 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.992 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.992 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 16384 2024-03-07T03:22:54.992 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.992 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.992 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 16384 2024-03-07T03:22:54.993 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.993 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.993 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:54.993 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:54.993 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 49152 2024-03-07T03:22:54.993 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.993 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.993 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 49152 2024-03-07T03:22:54.993 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.993 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.993 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 65536 2024-03-07T03:22:54.994 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.994 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.994 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 65536 2024-03-07T03:22:54.994 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.994 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.994 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:54.994 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:54.994 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 98304 2024-03-07T03:22:54.994 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.994 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.994 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 98304 2024-03-07T03:22:54.995 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.995 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.995 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 114688 2024-03-07T03:22:54.995 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.995 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.995 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 114688 2024-03-07T03:22:54.995 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.995 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.995 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:54.995 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:54.995 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 147456 2024-03-07T03:22:54.995 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.996 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.996 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 147456 2024-03-07T03:22:54.996 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.996 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.996 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 163840 2024-03-07T03:22:54.996 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.996 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.996 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 163840 2024-03-07T03:22:54.996 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.996 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.996 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:54.996 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:54.997 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 196608 2024-03-07T03:22:54.997 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.997 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.997 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 196608 2024-03-07T03:22:54.997 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.997 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.997 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 212992 2024-03-07T03:22:54.997 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.997 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.997 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 212992 2024-03-07T03:22:54.997 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.997 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.997 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:54.998 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.999 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.999 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.999 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:54.999 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.999 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:54.999 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:54.999 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:54.999 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:54.999 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:54.999 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:54.999 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:54.999 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:55.000 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:55.000 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:55.000 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:55.000 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:55.000 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:55.000 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:55.000 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:55.000 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:55.000 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:22:55.000 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:55.000 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:55.001 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:55.001 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:55.001 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:55.001 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:55.001 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:55.001 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:55.001 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:22:55.001 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:22:55.001 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:22:55.001 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:22:55.001 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:22:55.001 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:22:55.002 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:22:56.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:55 smithi145 bash[82329]: cluster 2024-03-07T03:22:54.758314+0000 mgr.y (mgr.34107) 332 : cluster 0 pgmap v613: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.5 MiB/s wr, 314 op/s; 358 KiB/s, 0 objects/s recovering 2024-03-07T03:22:56.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:55 smithi145 bash[79878]: cluster 2024-03-07T03:22:54.758314+0000 mgr.y (mgr.34107) 332 : cluster 0 pgmap v613: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.5 MiB/s wr, 314 op/s; 358 KiB/s, 0 objects/s recovering 2024-03-07T03:22:56.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:55 smithi189 bash[66969]: cluster 2024-03-07T03:22:54.758314+0000 mgr.y (mgr.34107) 332 : cluster 0 pgmap v613: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.5 MiB/s wr, 314 op/s; 358 KiB/s, 0 objects/s recovering 2024-03-07T03:22:58.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:57 smithi145 bash[79878]: cluster 2024-03-07T03:22:56.761007+0000 mgr.y (mgr.34107) 333 : cluster 0 pgmap v614: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.5 MiB/s wr, 313 op/s; 0 B/s, 0 objects/s recovering 2024-03-07T03:22:58.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:57 smithi145 bash[82329]: cluster 2024-03-07T03:22:56.761007+0000 mgr.y (mgr.34107) 333 : cluster 0 pgmap v614: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.5 MiB/s wr, 313 op/s; 0 B/s, 0 objects/s recovering 2024-03-07T03:22:58.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:57 smithi189 bash[66969]: cluster 2024-03-07T03:22:56.761007+0000 mgr.y (mgr.34107) 333 : cluster 0 pgmap v614: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.5 MiB/s wr, 313 op/s; 0 B/s, 0 objects/s recovering 2024-03-07T03:23:00.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:22:59 smithi145 bash[79878]: cluster 2024-03-07T03:22:58.763564+0000 mgr.y (mgr.34107) 334 : cluster 0 pgmap v615: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.6 MiB/s wr, 603 op/s 2024-03-07T03:23:00.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:22:59 smithi145 bash[82329]: cluster 2024-03-07T03:22:58.763564+0000 mgr.y (mgr.34107) 334 : cluster 0 pgmap v615: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.6 MiB/s wr, 603 op/s 2024-03-07T03:23:00.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:22:59 smithi189 bash[66969]: cluster 2024-03-07T03:22:58.763564+0000 mgr.y (mgr.34107) 334 : cluster 0 pgmap v615: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.6 MiB/s wr, 603 op/s 2024-03-07T03:23:02.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:01 smithi189 bash[66969]: cluster 2024-03-07T03:23:00.764815+0000 mgr.y (mgr.34107) 335 : cluster 0 pgmap v616: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.6 MiB/s wr, 605 op/s 2024-03-07T03:23:02.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:01 smithi145 bash[79878]: cluster 2024-03-07T03:23:00.764815+0000 mgr.y (mgr.34107) 335 : cluster 0 pgmap v616: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.6 MiB/s wr, 605 op/s 2024-03-07T03:23:02.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:01 smithi145 bash[82329]: cluster 2024-03-07T03:23:00.764815+0000 mgr.y (mgr.34107) 335 : cluster 0 pgmap v616: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 1.6 MiB/s wr, 605 op/s 2024-03-07T03:23:02.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:23:02 smithi189 bash[59814]: ts=2024-03-07T03:23:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:23:03.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:02 smithi145 bash[79878]: audit 2024-03-07T03:23:02.642834+0000 mon.a (mon.0) 1051 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:02 smithi145 bash[82329]: audit 2024-03-07T03:23:02.642834+0000 mon.a (mon.0) 1051 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:03.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:23:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:23:02] "GET /metrics HTTP/1.1" 200 60642 "" "Prometheus/2.43.0" 2024-03-07T03:23:03.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:02 smithi189 bash[66969]: audit 2024-03-07T03:23:02.642834+0000 mon.a (mon.0) 1051 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:04.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[82329]: cluster 2024-03-07T03:23:02.766071+0000 mgr.y (mgr.34107) 336 : cluster 0 pgmap v617: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 996 KiB/s wr, 606 op/s 2024-03-07T03:23:04.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[82329]: audit 2024-03-07T03:23:02.898979+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]: dispatch 2024-03-07T03:23:04.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[82329]: audit 2024-03-07T03:23:02.899275+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 5]}]: dispatch 2024-03-07T03:23:04.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[82329]: audit 2024-03-07T03:23:02.899509+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1", "id": [7, 0]}]: dispatch 2024-03-07T03:23:04.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[82329]: audit 2024-03-07T03:23:02.899734+0000 mon.a (mon.0) 1055 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]}]: dispatch 2024-03-07T03:23:04.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[82329]: audit 2024-03-07T03:23:02.899949+0000 mon.a (mon.0) 1056 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]}]: dispatch 2024-03-07T03:23:04.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[82329]: audit 2024-03-07T03:23:02.900163+0000 mon.a (mon.0) 1057 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.d", "id": [0, 2]}]: dispatch 2024-03-07T03:23:04.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[82329]: audit 2024-03-07T03:23:02.900380+0000 mon.a (mon.0) 1058 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]: dispatch 2024-03-07T03:23:04.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[82329]: audit 2024-03-07T03:23:02.900601+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.2", "id": [6, 5]}]: dispatch 2024-03-07T03:23:04.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[82329]: audit 2024-03-07T03:23:02.900848+0000 mon.a (mon.0) 1060 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.4", "id": [6, 4]}]: dispatch 2024-03-07T03:23:04.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[82329]: audit 2024-03-07T03:23:02.901087+0000 mon.a (mon.0) 1061 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.10", "id": [0, 1]}]: dispatch 2024-03-07T03:23:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[79878]: cluster 2024-03-07T03:23:02.766071+0000 mgr.y (mgr.34107) 336 : cluster 0 pgmap v617: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 996 KiB/s wr, 606 op/s 2024-03-07T03:23:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[79878]: audit 2024-03-07T03:23:02.898979+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]: dispatch 2024-03-07T03:23:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[79878]: audit 2024-03-07T03:23:02.899275+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 5]}]: dispatch 2024-03-07T03:23:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[79878]: audit 2024-03-07T03:23:02.899509+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1", "id": [7, 0]}]: dispatch 2024-03-07T03:23:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[79878]: audit 2024-03-07T03:23:02.899734+0000 mon.a (mon.0) 1055 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]}]: dispatch 2024-03-07T03:23:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[79878]: audit 2024-03-07T03:23:02.899949+0000 mon.a (mon.0) 1056 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]}]: dispatch 2024-03-07T03:23:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[79878]: audit 2024-03-07T03:23:02.900163+0000 mon.a (mon.0) 1057 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.d", "id": [0, 2]}]: dispatch 2024-03-07T03:23:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[79878]: audit 2024-03-07T03:23:02.900380+0000 mon.a (mon.0) 1058 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]: dispatch 2024-03-07T03:23:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[79878]: audit 2024-03-07T03:23:02.900601+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.2", "id": [6, 5]}]: dispatch 2024-03-07T03:23:04.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[79878]: audit 2024-03-07T03:23:02.900848+0000 mon.a (mon.0) 1060 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.4", "id": [6, 4]}]: dispatch 2024-03-07T03:23:04.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:03 smithi145 bash[79878]: audit 2024-03-07T03:23:02.901087+0000 mon.a (mon.0) 1061 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.10", "id": [0, 1]}]: dispatch 2024-03-07T03:23:04.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:03 smithi189 bash[66969]: cluster 2024-03-07T03:23:02.766071+0000 mgr.y (mgr.34107) 336 : cluster 0 pgmap v617: 425 pgs: 425 active+clean; 27 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 996 KiB/s wr, 606 op/s 2024-03-07T03:23:04.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:03 smithi189 bash[66969]: audit 2024-03-07T03:23:02.898979+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]: dispatch 2024-03-07T03:23:04.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:03 smithi189 bash[66969]: audit 2024-03-07T03:23:02.899275+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 5]}]: dispatch 2024-03-07T03:23:04.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:03 smithi189 bash[66969]: audit 2024-03-07T03:23:02.899509+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1", "id": [7, 0]}]: dispatch 2024-03-07T03:23:04.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:03 smithi189 bash[66969]: audit 2024-03-07T03:23:02.899734+0000 mon.a (mon.0) 1055 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]}]: dispatch 2024-03-07T03:23:04.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:03 smithi189 bash[66969]: audit 2024-03-07T03:23:02.899949+0000 mon.a (mon.0) 1056 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]}]: dispatch 2024-03-07T03:23:04.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:03 smithi189 bash[66969]: audit 2024-03-07T03:23:02.900163+0000 mon.a (mon.0) 1057 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.d", "id": [0, 2]}]: dispatch 2024-03-07T03:23:04.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:03 smithi189 bash[66969]: audit 2024-03-07T03:23:02.900380+0000 mon.a (mon.0) 1058 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]: dispatch 2024-03-07T03:23:04.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:03 smithi189 bash[66969]: audit 2024-03-07T03:23:02.900601+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.2", "id": [6, 5]}]: dispatch 2024-03-07T03:23:04.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:03 smithi189 bash[66969]: audit 2024-03-07T03:23:02.900848+0000 mon.a (mon.0) 1060 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.4", "id": [6, 4]}]: dispatch 2024-03-07T03:23:04.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:03 smithi189 bash[66969]: audit 2024-03-07T03:23:02.901087+0000 mon.a (mon.0) 1061 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.10", "id": [0, 1]}]: dispatch 2024-03-07T03:23:04.336 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.336 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.336 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.336 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.336 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.336 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.336 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.336 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.336 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.336 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.336 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.336 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.337 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.338 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.339 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.340 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.341 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.342 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:23:04.343 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:23:04.344 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:23:04.344 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:23:04.344 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:23:04.344 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:23:04.344 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 4 2024-03-07T03:23:04.344 INFO:tasks.workunit.client.0.smithi145.stdout:read: 4 2024-03-07T03:23:04.344 INFO:tasks.workunit.client.0.smithi145.stdout:read: 4 2024-03-07T03:23:04.344 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS2 (22421 ms) 2024-03-07T03:23:04.344 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCloneEncryption 2024-03-07T03:23:04.344 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:23:05.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[82329]: audit 2024-03-07T03:23:03.893722+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]': finished 2024-03-07T03:23:05.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[82329]: audit 2024-03-07T03:23:03.893819+0000 mon.a (mon.0) 1063 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 5]}]': finished 2024-03-07T03:23:05.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[82329]: audit 2024-03-07T03:23:03.893883+0000 mon.a (mon.0) 1064 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1", "id": [7, 0]}]': finished 2024-03-07T03:23:05.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[82329]: audit 2024-03-07T03:23:03.893953+0000 mon.a (mon.0) 1065 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]}]': finished 2024-03-07T03:23:05.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[82329]: audit 2024-03-07T03:23:03.894133+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]}]': finished 2024-03-07T03:23:05.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[82329]: audit 2024-03-07T03:23:03.894212+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.d", "id": [0, 2]}]': finished 2024-03-07T03:23:05.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[82329]: audit 2024-03-07T03:23:03.894330+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]': finished 2024-03-07T03:23:05.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[82329]: audit 2024-03-07T03:23:03.894436+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.2", "id": [6, 5]}]': finished 2024-03-07T03:23:05.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[82329]: audit 2024-03-07T03:23:03.894544+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.4", "id": [6, 4]}]': finished 2024-03-07T03:23:05.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[82329]: audit 2024-03-07T03:23:03.894641+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.10", "id": [0, 1]}]': finished 2024-03-07T03:23:05.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[82329]: cluster 2024-03-07T03:23:03.898421+0000 mon.a (mon.0) 1072 : cluster 0 osdmap e657: 8 total, 8 up, 8 in 2024-03-07T03:23:05.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[82329]: cluster 2024-03-07T03:23:04.766974+0000 mgr.y (mgr.34107) 337 : cluster 0 pgmap v619: 425 pgs: 425 active+clean; 31 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 238 KiB/s wr, 494 op/s 2024-03-07T03:23:05.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[79878]: audit 2024-03-07T03:23:03.893722+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]': finished 2024-03-07T03:23:05.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[79878]: audit 2024-03-07T03:23:03.893819+0000 mon.a (mon.0) 1063 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 5]}]': finished 2024-03-07T03:23:05.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[79878]: audit 2024-03-07T03:23:03.893883+0000 mon.a (mon.0) 1064 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1", "id": [7, 0]}]': finished 2024-03-07T03:23:05.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[79878]: audit 2024-03-07T03:23:03.893953+0000 mon.a (mon.0) 1065 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]}]': finished 2024-03-07T03:23:05.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[79878]: audit 2024-03-07T03:23:03.894133+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]}]': finished 2024-03-07T03:23:05.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[79878]: audit 2024-03-07T03:23:03.894212+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.d", "id": [0, 2]}]': finished 2024-03-07T03:23:05.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[79878]: audit 2024-03-07T03:23:03.894330+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]': finished 2024-03-07T03:23:05.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[79878]: audit 2024-03-07T03:23:03.894436+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.2", "id": [6, 5]}]': finished 2024-03-07T03:23:05.314 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[79878]: audit 2024-03-07T03:23:03.894544+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.4", "id": [6, 4]}]': finished 2024-03-07T03:23:05.314 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[79878]: audit 2024-03-07T03:23:03.894641+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.10", "id": [0, 1]}]': finished 2024-03-07T03:23:05.314 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[79878]: cluster 2024-03-07T03:23:03.898421+0000 mon.a (mon.0) 1072 : cluster 0 osdmap e657: 8 total, 8 up, 8 in 2024-03-07T03:23:05.314 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:04 smithi145 bash[79878]: cluster 2024-03-07T03:23:04.766974+0000 mgr.y (mgr.34107) 337 : cluster 0 pgmap v619: 425 pgs: 425 active+clean; 31 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 238 KiB/s wr, 494 op/s 2024-03-07T03:23:05.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:04 smithi189 bash[66969]: audit 2024-03-07T03:23:03.893722+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]': finished 2024-03-07T03:23:05.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:04 smithi189 bash[66969]: audit 2024-03-07T03:23:03.893819+0000 mon.a (mon.0) 1063 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 5]}]': finished 2024-03-07T03:23:05.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:04 smithi189 bash[66969]: audit 2024-03-07T03:23:03.893883+0000 mon.a (mon.0) 1064 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1", "id": [7, 0]}]': finished 2024-03-07T03:23:05.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:04 smithi189 bash[66969]: audit 2024-03-07T03:23:03.893953+0000 mon.a (mon.0) 1065 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.a", "id": [7, 0]}]': finished 2024-03-07T03:23:05.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:04 smithi189 bash[66969]: audit 2024-03-07T03:23:03.894133+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]}]': finished 2024-03-07T03:23:05.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:04 smithi189 bash[66969]: audit 2024-03-07T03:23:03.894212+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.d", "id": [0, 2]}]': finished 2024-03-07T03:23:05.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:04 smithi189 bash[66969]: audit 2024-03-07T03:23:03.894330+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]': finished 2024-03-07T03:23:05.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:04 smithi189 bash[66969]: audit 2024-03-07T03:23:03.894436+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.2", "id": [6, 5]}]': finished 2024-03-07T03:23:05.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:04 smithi189 bash[66969]: audit 2024-03-07T03:23:03.894544+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.4", "id": [6, 4]}]': finished 2024-03-07T03:23:05.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:04 smithi189 bash[66969]: audit 2024-03-07T03:23:03.894641+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.10", "id": [0, 1]}]': finished 2024-03-07T03:23:05.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:04 smithi189 bash[66969]: cluster 2024-03-07T03:23:03.898421+0000 mon.a (mon.0) 1072 : cluster 0 osdmap e657: 8 total, 8 up, 8 in 2024-03-07T03:23:05.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:04 smithi189 bash[66969]: cluster 2024-03-07T03:23:04.766974+0000 mgr.y (mgr.34107) 337 : cluster 0 pgmap v619: 425 pgs: 425 active+clean; 31 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 238 KiB/s wr, 494 op/s 2024-03-07T03:23:06.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:05 smithi145 bash[79878]: cluster 2024-03-07T03:23:04.913017+0000 mon.a (mon.0) 1073 : cluster 0 osdmap e658: 8 total, 8 up, 8 in 2024-03-07T03:23:06.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:05 smithi145 bash[82329]: cluster 2024-03-07T03:23:04.913017+0000 mon.a (mon.0) 1073 : cluster 0 osdmap e658: 8 total, 8 up, 8 in 2024-03-07T03:23:06.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:05 smithi189 bash[66969]: cluster 2024-03-07T03:23:04.913017+0000 mon.a (mon.0) 1073 : cluster 0 osdmap e658: 8 total, 8 up, 8 in 2024-03-07T03:23:07.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:06 smithi145 bash[82329]: cluster 2024-03-07T03:23:06.769126+0000 mgr.y (mgr.34107) 338 : cluster 0 pgmap v621: 425 pgs: 2 peering, 423 active+clean; 31 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 80 KiB/s wr, 193 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:07.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:06 smithi145 bash[79878]: cluster 2024-03-07T03:23:06.769126+0000 mgr.y (mgr.34107) 338 : cluster 0 pgmap v621: 425 pgs: 2 peering, 423 active+clean; 31 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 80 KiB/s wr, 193 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:07.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:06 smithi189 bash[66969]: cluster 2024-03-07T03:23:06.769126+0000 mgr.y (mgr.34107) 338 : cluster 0 pgmap v621: 425 pgs: 2 peering, 423 active+clean; 31 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 80 KiB/s wr, 193 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:08.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:07 smithi145 bash[82329]: cluster 2024-03-07T03:23:06.906770+0000 mon.a (mon.0) 1074 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:23:08.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:07 smithi145 bash[79878]: cluster 2024-03-07T03:23:06.906770+0000 mon.a (mon.0) 1074 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:23:08.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:07 smithi189 bash[66969]: cluster 2024-03-07T03:23:06.906770+0000 mon.a (mon.0) 1074 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:23:09.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:08 smithi145 bash[79878]: cluster 2024-03-07T03:23:08.771602+0000 mgr.y (mgr.34107) 339 : cluster 0 pgmap v622: 425 pgs: 2 peering, 423 active+clean; 31 MiB data, 452 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 81 KiB/s wr, 211 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:09.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:08 smithi145 bash[82329]: cluster 2024-03-07T03:23:08.771602+0000 mgr.y (mgr.34107) 339 : cluster 0 pgmap v622: 425 pgs: 2 peering, 423 active+clean; 31 MiB data, 452 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 81 KiB/s wr, 211 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:09.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:08 smithi189 bash[66969]: cluster 2024-03-07T03:23:08.771602+0000 mgr.y (mgr.34107) 339 : cluster 0 pgmap v622: 425 pgs: 2 peering, 423 active+clean; 31 MiB data, 452 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 81 KiB/s wr, 211 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:11.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:10 smithi145 bash[79878]: cluster 2024-03-07T03:23:10.825571+0000 mon.a (mon.0) 1075 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-07T03:23:11.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:10 smithi145 bash[82329]: cluster 2024-03-07T03:23:10.825571+0000 mon.a (mon.0) 1075 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-07T03:23:11.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:10 smithi189 bash[66969]: cluster 2024-03-07T03:23:10.825571+0000 mon.a (mon.0) 1075 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-07T03:23:12.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:11 smithi189 bash[66969]: cluster 2024-03-07T03:23:10.773438+0000 mgr.y (mgr.34107) 340 : cluster 0 pgmap v623: 425 pgs: 425 active+clean; 31 MiB data, 456 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 81 KiB/s wr, 211 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:12.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:11 smithi145 bash[82329]: cluster 2024-03-07T03:23:10.773438+0000 mgr.y (mgr.34107) 340 : cluster 0 pgmap v623: 425 pgs: 425 active+clean; 31 MiB data, 456 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 81 KiB/s wr, 211 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:12.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:11 smithi145 bash[79878]: cluster 2024-03-07T03:23:10.773438+0000 mgr.y (mgr.34107) 340 : cluster 0 pgmap v623: 425 pgs: 425 active+clean; 31 MiB data, 456 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 81 KiB/s wr, 211 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:23:12 smithi189 bash[59814]: ts=2024-03-07T03:23:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:23:13.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:12 smithi145 bash[79878]: cluster 2024-03-07T03:23:11.897775+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e659: 8 total, 8 up, 8 in 2024-03-07T03:23:13.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:12 smithi145 bash[79878]: cluster 2024-03-07T03:23:12.774484+0000 mgr.y (mgr.34107) 341 : cluster 0 pgmap v625: 425 pgs: 425 active+clean; 35 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 612 KiB/s rd, 514 KiB/s wr, 38 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:13.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:23:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:23:12] "GET /metrics HTTP/1.1" 200 60642 "" "Prometheus/2.43.0" 2024-03-07T03:23:13.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:12 smithi145 bash[82329]: cluster 2024-03-07T03:23:11.897775+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e659: 8 total, 8 up, 8 in 2024-03-07T03:23:13.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:12 smithi145 bash[82329]: cluster 2024-03-07T03:23:12.774484+0000 mgr.y (mgr.34107) 341 : cluster 0 pgmap v625: 425 pgs: 425 active+clean; 35 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 612 KiB/s rd, 514 KiB/s wr, 38 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:13.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:12 smithi189 bash[66969]: cluster 2024-03-07T03:23:11.897775+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e659: 8 total, 8 up, 8 in 2024-03-07T03:23:13.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:12 smithi189 bash[66969]: cluster 2024-03-07T03:23:12.774484+0000 mgr.y (mgr.34107) 341 : cluster 0 pgmap v625: 425 pgs: 425 active+clean; 35 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 612 KiB/s rd, 514 KiB/s wr, 38 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:16.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:15 smithi189 bash[66969]: cluster 2024-03-07T03:23:14.775977+0000 mgr.y (mgr.34107) 342 : cluster 0 pgmap v626: 425 pgs: 425 active+clean; 35 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 505 KiB/s rd, 418 KiB/s wr, 41 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:16.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:15 smithi145 bash[79878]: cluster 2024-03-07T03:23:14.775977+0000 mgr.y (mgr.34107) 342 : cluster 0 pgmap v626: 425 pgs: 425 active+clean; 35 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 505 KiB/s rd, 418 KiB/s wr, 41 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:16.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:15 smithi145 bash[82329]: cluster 2024-03-07T03:23:14.775977+0000 mgr.y (mgr.34107) 342 : cluster 0 pgmap v626: 425 pgs: 425 active+clean; 35 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 505 KiB/s rd, 418 KiB/s wr, 41 op/s; 1 B/s, 0 objects/s recovering 2024-03-07T03:23:18.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:17 smithi189 bash[66969]: cluster 2024-03-07T03:23:16.777586+0000 mgr.y (mgr.34107) 343 : cluster 0 pgmap v627: 425 pgs: 425 active+clean; 35 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s rd, 412 KiB/s wr, 48 op/s 2024-03-07T03:23:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:17 smithi189 bash[66969]: audit 2024-03-07T03:23:17.644153+0000 mon.a (mon.0) 1077 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:18.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:17 smithi145 bash[79878]: cluster 2024-03-07T03:23:16.777586+0000 mgr.y (mgr.34107) 343 : cluster 0 pgmap v627: 425 pgs: 425 active+clean; 35 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s rd, 412 KiB/s wr, 48 op/s 2024-03-07T03:23:18.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:17 smithi145 bash[79878]: audit 2024-03-07T03:23:17.644153+0000 mon.a (mon.0) 1077 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:18.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:17 smithi145 bash[82329]: cluster 2024-03-07T03:23:16.777586+0000 mgr.y (mgr.34107) 343 : cluster 0 pgmap v627: 425 pgs: 425 active+clean; 35 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s rd, 412 KiB/s wr, 48 op/s 2024-03-07T03:23:18.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:17 smithi145 bash[82329]: audit 2024-03-07T03:23:17.644153+0000 mon.a (mon.0) 1077 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:20.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:19 smithi145 bash[82329]: cluster 2024-03-07T03:23:18.779088+0000 mgr.y (mgr.34107) 344 : cluster 0 pgmap v628: 425 pgs: 425 active+clean; 43 MiB data, 480 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-03-07T03:23:20.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:19 smithi145 bash[82329]: audit 2024-03-07T03:23:18.870078+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:20.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:19 smithi145 bash[79878]: cluster 2024-03-07T03:23:18.779088+0000 mgr.y (mgr.34107) 344 : cluster 0 pgmap v628: 425 pgs: 425 active+clean; 43 MiB data, 480 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-03-07T03:23:20.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:19 smithi145 bash[79878]: audit 2024-03-07T03:23:18.870078+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:20.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:19 smithi189 bash[66969]: cluster 2024-03-07T03:23:18.779088+0000 mgr.y (mgr.34107) 344 : cluster 0 pgmap v628: 425 pgs: 425 active+clean; 43 MiB data, 480 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-03-07T03:23:20.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:19 smithi189 bash[66969]: audit 2024-03-07T03:23:18.870078+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:22.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:21 smithi189 bash[66969]: cluster 2024-03-07T03:23:20.780752+0000 mgr.y (mgr.34107) 345 : cluster 0 pgmap v629: 425 pgs: 425 active+clean; 51 MiB data, 500 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 2.0 MiB/s wr, 31 op/s 2024-03-07T03:23:22.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:21 smithi145 bash[82329]: cluster 2024-03-07T03:23:20.780752+0000 mgr.y (mgr.34107) 345 : cluster 0 pgmap v629: 425 pgs: 425 active+clean; 51 MiB data, 500 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 2.0 MiB/s wr, 31 op/s 2024-03-07T03:23:22.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:21 smithi145 bash[79878]: cluster 2024-03-07T03:23:20.780752+0000 mgr.y (mgr.34107) 345 : cluster 0 pgmap v629: 425 pgs: 425 active+clean; 51 MiB data, 500 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 2.0 MiB/s wr, 31 op/s 2024-03-07T03:23:22.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:23:22 smithi189 bash[59814]: ts=2024-03-07T03:23:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:23:23.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:22 smithi145 bash[79878]: cluster 2024-03-07T03:23:21.883802+0000 mon.a (mon.0) 1079 : cluster 0 osdmap e660: 8 total, 8 up, 8 in 2024-03-07T03:23:23.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:23:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:23:22] "GET /metrics HTTP/1.1" 200 60639 "" "Prometheus/2.43.0" 2024-03-07T03:23:23.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:22 smithi145 bash[82329]: cluster 2024-03-07T03:23:21.883802+0000 mon.a (mon.0) 1079 : cluster 0 osdmap e660: 8 total, 8 up, 8 in 2024-03-07T03:23:23.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:22 smithi189 bash[66969]: cluster 2024-03-07T03:23:21.883802+0000 mon.a (mon.0) 1079 : cluster 0 osdmap e660: 8 total, 8 up, 8 in 2024-03-07T03:23:24.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:23 smithi145 bash[79878]: cluster 2024-03-07T03:23:22.781686+0000 mgr.y (mgr.34107) 346 : cluster 0 pgmap v631: 425 pgs: 425 active+clean; 51 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 1.6 MiB/s wr, 34 op/s 2024-03-07T03:23:24.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:23 smithi145 bash[82329]: cluster 2024-03-07T03:23:22.781686+0000 mgr.y (mgr.34107) 346 : cluster 0 pgmap v631: 425 pgs: 425 active+clean; 51 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 1.6 MiB/s wr, 34 op/s 2024-03-07T03:23:24.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:23 smithi189 bash[66969]: cluster 2024-03-07T03:23:22.781686+0000 mgr.y (mgr.34107) 346 : cluster 0 pgmap v631: 425 pgs: 425 active+clean; 51 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 1.6 MiB/s wr, 34 op/s 2024-03-07T03:23:25.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:24 smithi145 bash[79878]: cluster 2024-03-07T03:23:24.783048+0000 mgr.y (mgr.34107) 347 : cluster 0 pgmap v632: 425 pgs: 425 active+clean; 55 MiB data, 516 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 2.0 MiB/s wr, 34 op/s 2024-03-07T03:23:25.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:24 smithi145 bash[82329]: cluster 2024-03-07T03:23:24.783048+0000 mgr.y (mgr.34107) 347 : cluster 0 pgmap v632: 425 pgs: 425 active+clean; 55 MiB data, 516 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 2.0 MiB/s wr, 34 op/s 2024-03-07T03:23:25.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:24 smithi189 bash[66969]: cluster 2024-03-07T03:23:24.783048+0000 mgr.y (mgr.34107) 347 : cluster 0 pgmap v632: 425 pgs: 425 active+clean; 55 MiB data, 516 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 2.0 MiB/s wr, 34 op/s 2024-03-07T03:23:28.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:27 smithi189 bash[66969]: cluster 2024-03-07T03:23:26.784635+0000 mgr.y (mgr.34107) 348 : cluster 0 pgmap v633: 425 pgs: 425 active+clean; 55 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 2.0 MiB/s wr, 40 op/s 2024-03-07T03:23:28.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:27 smithi145 bash[79878]: cluster 2024-03-07T03:23:26.784635+0000 mgr.y (mgr.34107) 348 : cluster 0 pgmap v633: 425 pgs: 425 active+clean; 55 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 2.0 MiB/s wr, 40 op/s 2024-03-07T03:23:28.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:27 smithi145 bash[82329]: cluster 2024-03-07T03:23:26.784635+0000 mgr.y (mgr.34107) 348 : cluster 0 pgmap v633: 425 pgs: 425 active+clean; 55 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 2.0 MiB/s wr, 40 op/s 2024-03-07T03:23:30.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:29 smithi145 bash[79878]: cluster 2024-03-07T03:23:28.786171+0000 mgr.y (mgr.34107) 349 : cluster 0 pgmap v634: 425 pgs: 425 active+clean; 55 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 1.2 MiB/s wr, 40 op/s 2024-03-07T03:23:30.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:29 smithi145 bash[82329]: cluster 2024-03-07T03:23:28.786171+0000 mgr.y (mgr.34107) 349 : cluster 0 pgmap v634: 425 pgs: 425 active+clean; 55 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 1.2 MiB/s wr, 40 op/s 2024-03-07T03:23:30.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:29 smithi189 bash[66969]: cluster 2024-03-07T03:23:28.786171+0000 mgr.y (mgr.34107) 349 : cluster 0 pgmap v634: 425 pgs: 425 active+clean; 55 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 1.2 MiB/s wr, 40 op/s 2024-03-07T03:23:32.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:31 smithi189 bash[66969]: cluster 2024-03-07T03:23:30.787254+0000 mgr.y (mgr.34107) 350 : cluster 0 pgmap v635: 425 pgs: 425 active+clean; 59 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 821 KiB/s wr, 40 op/s 2024-03-07T03:23:32.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:31 smithi145 bash[79878]: cluster 2024-03-07T03:23:30.787254+0000 mgr.y (mgr.34107) 350 : cluster 0 pgmap v635: 425 pgs: 425 active+clean; 59 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 821 KiB/s wr, 40 op/s 2024-03-07T03:23:32.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:31 smithi145 bash[82329]: cluster 2024-03-07T03:23:30.787254+0000 mgr.y (mgr.34107) 350 : cluster 0 pgmap v635: 425 pgs: 425 active+clean; 59 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 821 KiB/s wr, 40 op/s 2024-03-07T03:23:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:23:32 smithi189 bash[59814]: ts=2024-03-07T03:23:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:23:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:32 smithi145 bash[79878]: audit 2024-03-07T03:23:32.643796+0000 mon.a (mon.0) 1080 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:33.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:23:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:23:32] "GET /metrics HTTP/1.1" 200 60651 "" "Prometheus/2.43.0" 2024-03-07T03:23:33.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:32 smithi145 bash[82329]: audit 2024-03-07T03:23:32.643796+0000 mon.a (mon.0) 1080 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:33.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:32 smithi189 bash[66969]: audit 2024-03-07T03:23:32.643796+0000 mon.a (mon.0) 1080 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:34.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:33 smithi145 bash[79878]: cluster 2024-03-07T03:23:32.788559+0000 mgr.y (mgr.34107) 351 : cluster 0 pgmap v636: 425 pgs: 425 active+clean; 67 MiB data, 556 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 1.5 MiB/s wr, 37 op/s 2024-03-07T03:23:34.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:33 smithi145 bash[82329]: cluster 2024-03-07T03:23:32.788559+0000 mgr.y (mgr.34107) 351 : cluster 0 pgmap v636: 425 pgs: 425 active+clean; 67 MiB data, 556 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 1.5 MiB/s wr, 37 op/s 2024-03-07T03:23:34.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:33 smithi189 bash[66969]: cluster 2024-03-07T03:23:32.788559+0000 mgr.y (mgr.34107) 351 : cluster 0 pgmap v636: 425 pgs: 425 active+clean; 67 MiB data, 556 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 1.5 MiB/s wr, 37 op/s 2024-03-07T03:23:36.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:35 smithi145 bash[79878]: cluster 2024-03-07T03:23:34.789848+0000 mgr.y (mgr.34107) 352 : cluster 0 pgmap v637: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 732 KiB/s rd, 1.7 MiB/s wr, 34 op/s 2024-03-07T03:23:36.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:35 smithi145 bash[82329]: cluster 2024-03-07T03:23:34.789848+0000 mgr.y (mgr.34107) 352 : cluster 0 pgmap v637: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 732 KiB/s rd, 1.7 MiB/s wr, 34 op/s 2024-03-07T03:23:36.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:35 smithi189 bash[66969]: cluster 2024-03-07T03:23:34.789848+0000 mgr.y (mgr.34107) 352 : cluster 0 pgmap v637: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 732 KiB/s rd, 1.7 MiB/s wr, 34 op/s 2024-03-07T03:23:38.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:37 smithi145 bash[79878]: cluster 2024-03-07T03:23:36.791451+0000 mgr.y (mgr.34107) 353 : cluster 0 pgmap v638: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 26 op/s 2024-03-07T03:23:38.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:37 smithi145 bash[82329]: cluster 2024-03-07T03:23:36.791451+0000 mgr.y (mgr.34107) 353 : cluster 0 pgmap v638: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 26 op/s 2024-03-07T03:23:38.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:37 smithi189 bash[66969]: cluster 2024-03-07T03:23:36.791451+0000 mgr.y (mgr.34107) 353 : cluster 0 pgmap v638: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 26 op/s 2024-03-07T03:23:40.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:39 smithi145 bash[79878]: cluster 2024-03-07T03:23:38.793648+0000 mgr.y (mgr.34107) 354 : cluster 0 pgmap v639: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-03-07T03:23:40.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:39 smithi145 bash[82329]: cluster 2024-03-07T03:23:38.793648+0000 mgr.y (mgr.34107) 354 : cluster 0 pgmap v639: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-03-07T03:23:40.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:39 smithi189 bash[66969]: cluster 2024-03-07T03:23:38.793648+0000 mgr.y (mgr.34107) 354 : cluster 0 pgmap v639: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-03-07T03:23:42.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:41 smithi189 bash[66969]: cluster 2024-03-07T03:23:40.795435+0000 mgr.y (mgr.34107) 355 : cluster 0 pgmap v640: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-03-07T03:23:42.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:41 smithi189 bash[66969]: cluster 2024-03-07T03:23:40.883025+0000 mon.a (mon.0) 1081 : cluster 0 osdmap e661: 8 total, 8 up, 8 in 2024-03-07T03:23:42.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:41 smithi145 bash[79878]: cluster 2024-03-07T03:23:40.795435+0000 mgr.y (mgr.34107) 355 : cluster 0 pgmap v640: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-03-07T03:23:42.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:41 smithi145 bash[79878]: cluster 2024-03-07T03:23:40.883025+0000 mon.a (mon.0) 1081 : cluster 0 osdmap e661: 8 total, 8 up, 8 in 2024-03-07T03:23:42.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:41 smithi145 bash[82329]: cluster 2024-03-07T03:23:40.795435+0000 mgr.y (mgr.34107) 355 : cluster 0 pgmap v640: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-03-07T03:23:42.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:41 smithi145 bash[82329]: cluster 2024-03-07T03:23:40.883025+0000 mon.a (mon.0) 1081 : cluster 0 osdmap e661: 8 total, 8 up, 8 in 2024-03-07T03:23:42.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:23:42 smithi189 bash[59814]: ts=2024-03-07T03:23:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:23:43.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:23:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:23:42] "GET /metrics HTTP/1.1" 200 60651 "" "Prometheus/2.43.0" 2024-03-07T03:23:44.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:43 smithi145 bash[79878]: cluster 2024-03-07T03:23:42.797135+0000 mgr.y (mgr.34107) 356 : cluster 0 pgmap v642: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 410 KiB/s wr, 20 op/s 2024-03-07T03:23:44.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:43 smithi145 bash[82329]: cluster 2024-03-07T03:23:42.797135+0000 mgr.y (mgr.34107) 356 : cluster 0 pgmap v642: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 410 KiB/s wr, 20 op/s 2024-03-07T03:23:44.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:43 smithi189 bash[66969]: cluster 2024-03-07T03:23:42.797135+0000 mgr.y (mgr.34107) 356 : cluster 0 pgmap v642: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 410 KiB/s wr, 20 op/s 2024-03-07T03:23:46.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:45 smithi145 bash[79878]: cluster 2024-03-07T03:23:44.798557+0000 mgr.y (mgr.34107) 357 : cluster 0 pgmap v643: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 523 KiB/s rd, 1023 B/s wr, 34 op/s 2024-03-07T03:23:46.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:45 smithi145 bash[82329]: cluster 2024-03-07T03:23:44.798557+0000 mgr.y (mgr.34107) 357 : cluster 0 pgmap v643: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 523 KiB/s rd, 1023 B/s wr, 34 op/s 2024-03-07T03:23:46.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:45 smithi189 bash[66969]: cluster 2024-03-07T03:23:44.798557+0000 mgr.y (mgr.34107) 357 : cluster 0 pgmap v643: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 523 KiB/s rd, 1023 B/s wr, 34 op/s 2024-03-07T03:23:48.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:47 smithi145 bash[79878]: cluster 2024-03-07T03:23:46.799880+0000 mgr.y (mgr.34107) 358 : cluster 0 pgmap v644: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 1.1 KiB/s wr, 42 op/s 2024-03-07T03:23:48.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:47 smithi145 bash[79878]: audit 2024-03-07T03:23:47.644931+0000 mon.a (mon.0) 1082 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:48.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:47 smithi145 bash[82329]: cluster 2024-03-07T03:23:46.799880+0000 mgr.y (mgr.34107) 358 : cluster 0 pgmap v644: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 1.1 KiB/s wr, 42 op/s 2024-03-07T03:23:48.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:47 smithi145 bash[82329]: audit 2024-03-07T03:23:47.644931+0000 mon.a (mon.0) 1082 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:48.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:47 smithi189 bash[66969]: cluster 2024-03-07T03:23:46.799880+0000 mgr.y (mgr.34107) 358 : cluster 0 pgmap v644: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 1.1 KiB/s wr, 42 op/s 2024-03-07T03:23:48.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:47 smithi189 bash[66969]: audit 2024-03-07T03:23:47.644931+0000 mon.a (mon.0) 1082 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:49.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:48 smithi145 bash[79878]: cluster 2024-03-07T03:23:48.801392+0000 mgr.y (mgr.34107) 359 : cluster 0 pgmap v645: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 152 KiB/s rd, 1.7 KiB/s wr, 45 op/s 2024-03-07T03:23:49.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:48 smithi145 bash[82329]: cluster 2024-03-07T03:23:48.801392+0000 mgr.y (mgr.34107) 359 : cluster 0 pgmap v645: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 152 KiB/s rd, 1.7 KiB/s wr, 45 op/s 2024-03-07T03:23:49.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:48 smithi189 bash[66969]: cluster 2024-03-07T03:23:48.801392+0000 mgr.y (mgr.34107) 359 : cluster 0 pgmap v645: 425 pgs: 425 active+clean; 71 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 152 KiB/s rd, 1.7 KiB/s wr, 45 op/s 2024-03-07T03:23:50.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:49 smithi145 bash[79878]: audit 2024-03-07T03:23:49.003544+0000 mon.a (mon.0) 1083 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:23:50.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:49 smithi145 bash[82329]: audit 2024-03-07T03:23:49.003544+0000 mon.a (mon.0) 1083 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:23:50.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:49 smithi189 bash[66969]: audit 2024-03-07T03:23:49.003544+0000 mon.a (mon.0) 1083 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:23:51.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:51 smithi145 bash[79878]: cluster 2024-03-07T03:23:50.802716+0000 mgr.y (mgr.34107) 360 : cluster 0 pgmap v646: 425 pgs: 425 active+clean; 183 MiB data, 980 MiB used, 714 GiB / 715 GiB avail; 152 KiB/s rd, 11 MiB/s wr, 53 op/s 2024-03-07T03:23:51.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:51 smithi145 bash[82329]: cluster 2024-03-07T03:23:50.802716+0000 mgr.y (mgr.34107) 360 : cluster 0 pgmap v646: 425 pgs: 425 active+clean; 183 MiB data, 980 MiB used, 714 GiB / 715 GiB avail; 152 KiB/s rd, 11 MiB/s wr, 53 op/s 2024-03-07T03:23:51.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:51 smithi189 bash[66969]: cluster 2024-03-07T03:23:50.802716+0000 mgr.y (mgr.34107) 360 : cluster 0 pgmap v646: 425 pgs: 425 active+clean; 183 MiB data, 980 MiB used, 714 GiB / 715 GiB avail; 152 KiB/s rd, 11 MiB/s wr, 53 op/s 2024-03-07T03:23:52.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:23:52 smithi189 bash[59814]: ts=2024-03-07T03:23:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:23:53.003 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:23:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:23:52] "GET /metrics HTTP/1.1" 200 60644 "" "Prometheus/2.43.0" 2024-03-07T03:23:53.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:53 smithi145 bash[82329]: cluster 2024-03-07T03:23:52.009785+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e662: 8 total, 8 up, 8 in 2024-03-07T03:23:53.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:53 smithi145 bash[82329]: cluster 2024-03-07T03:23:52.803641+0000 mgr.y (mgr.34107) 361 : cluster 0 pgmap v648: 425 pgs: 425 active+clean; 183 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 521 KiB/s rd, 14 MiB/s wr, 44 op/s 2024-03-07T03:23:53.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:53 smithi145 bash[79878]: cluster 2024-03-07T03:23:52.009785+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e662: 8 total, 8 up, 8 in 2024-03-07T03:23:53.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:53 smithi145 bash[79878]: cluster 2024-03-07T03:23:52.803641+0000 mgr.y (mgr.34107) 361 : cluster 0 pgmap v648: 425 pgs: 425 active+clean; 183 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 521 KiB/s rd, 14 MiB/s wr, 44 op/s 2024-03-07T03:23:53.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:53 smithi189 bash[66969]: cluster 2024-03-07T03:23:52.009785+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e662: 8 total, 8 up, 8 in 2024-03-07T03:23:53.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:53 smithi189 bash[66969]: cluster 2024-03-07T03:23:52.803641+0000 mgr.y (mgr.34107) 361 : cluster 0 pgmap v648: 425 pgs: 425 active+clean; 183 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 521 KiB/s rd, 14 MiB/s wr, 44 op/s 2024-03-07T03:23:54.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:54 smithi145 bash[82329]: cluster 2024-03-07T03:23:53.023227+0000 mon.a (mon.0) 1085 : cluster 0 osdmap e663: 8 total, 8 up, 8 in 2024-03-07T03:23:54.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:54 smithi145 bash[79878]: cluster 2024-03-07T03:23:53.023227+0000 mon.a (mon.0) 1085 : cluster 0 osdmap e663: 8 total, 8 up, 8 in 2024-03-07T03:23:54.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:54 smithi189 bash[66969]: cluster 2024-03-07T03:23:53.023227+0000 mon.a (mon.0) 1085 : cluster 0 osdmap e663: 8 total, 8 up, 8 in 2024-03-07T03:23:55.169 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 4 2024-03-07T03:23:55.169 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 4 2024-03-07T03:23:55.169 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 4 2024-03-07T03:23:55.169 INFO:tasks.workunit.client.0.smithi145.stdout:read: 4 2024-03-07T03:23:55.169 INFO:tasks.workunit.client.0.smithi145.stdout:read: 4 2024-03-07T03:23:55.169 INFO:tasks.workunit.client.0.smithi145.stdout:read: 4 2024-03-07T03:23:55.169 INFO:tasks.workunit.client.0.smithi145.stdout:read: 4 2024-03-07T03:23:55.169 INFO:tasks.workunit.client.0.smithi145.stdout:read: 4 2024-03-07T03:23:55.169 INFO:tasks.workunit.client.0.smithi145.stdout:read: 4 2024-03-07T03:23:55.169 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCloneEncryption (50833 ms) 2024-03-07T03:23:55.170 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.LUKS1UnderLUKS2WithoutResize 2024-03-07T03:23:55.170 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:23:55.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:55 smithi145 bash[82329]: cluster 2024-03-07T03:23:54.028430+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e664: 8 total, 8 up, 8 in 2024-03-07T03:23:55.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:55 smithi145 bash[82329]: cluster 2024-03-07T03:23:54.804447+0000 mgr.y (mgr.34107) 362 : cluster 0 pgmap v651: 425 pgs: 425 active+clean; 167 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 43 MiB/s wr, 168 op/s 2024-03-07T03:23:55.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:55 smithi145 bash[82329]: audit 2024-03-07T03:23:54.952317+0000 mon.a (mon.0) 1087 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:55.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:55 smithi145 bash[82329]: audit 2024-03-07T03:23:54.960144+0000 mon.a (mon.0) 1088 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:55.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:55 smithi145 bash[79878]: cluster 2024-03-07T03:23:54.028430+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e664: 8 total, 8 up, 8 in 2024-03-07T03:23:55.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:55 smithi145 bash[79878]: cluster 2024-03-07T03:23:54.804447+0000 mgr.y (mgr.34107) 362 : cluster 0 pgmap v651: 425 pgs: 425 active+clean; 167 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 43 MiB/s wr, 168 op/s 2024-03-07T03:23:55.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:55 smithi145 bash[79878]: audit 2024-03-07T03:23:54.952317+0000 mon.a (mon.0) 1087 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:55.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:55 smithi145 bash[79878]: audit 2024-03-07T03:23:54.960144+0000 mon.a (mon.0) 1088 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:55.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:55 smithi189 bash[66969]: cluster 2024-03-07T03:23:54.028430+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e664: 8 total, 8 up, 8 in 2024-03-07T03:23:55.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:55 smithi189 bash[66969]: cluster 2024-03-07T03:23:54.804447+0000 mgr.y (mgr.34107) 362 : cluster 0 pgmap v651: 425 pgs: 425 active+clean; 167 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 43 MiB/s wr, 168 op/s 2024-03-07T03:23:55.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:55 smithi189 bash[66969]: audit 2024-03-07T03:23:54.952317+0000 mon.a (mon.0) 1087 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:55.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:55 smithi189 bash[66969]: audit 2024-03-07T03:23:54.960144+0000 mon.a (mon.0) 1088 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:56.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:56 smithi145 bash[79878]: cluster 2024-03-07T03:23:55.038398+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e665: 8 total, 8 up, 8 in 2024-03-07T03:23:56.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:56 smithi145 bash[79878]: audit 2024-03-07T03:23:55.106488+0000 mon.a (mon.0) 1090 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:56.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:56 smithi145 bash[79878]: audit 2024-03-07T03:23:55.116019+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:56.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:56 smithi145 bash[79878]: audit 2024-03-07T03:23:55.693586+0000 mon.a (mon.0) 1092 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:23:56.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:56 smithi145 bash[79878]: audit 2024-03-07T03:23:55.694933+0000 mon.a (mon.0) 1093 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:23:56.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:56 smithi145 bash[79878]: audit 2024-03-07T03:23:55.702470+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:56.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:56 smithi145 bash[82329]: cluster 2024-03-07T03:23:55.038398+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e665: 8 total, 8 up, 8 in 2024-03-07T03:23:56.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:56 smithi145 bash[82329]: audit 2024-03-07T03:23:55.106488+0000 mon.a (mon.0) 1090 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:56.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:56 smithi145 bash[82329]: audit 2024-03-07T03:23:55.116019+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:56.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:56 smithi145 bash[82329]: audit 2024-03-07T03:23:55.693586+0000 mon.a (mon.0) 1092 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:23:56.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:56 smithi145 bash[82329]: audit 2024-03-07T03:23:55.694933+0000 mon.a (mon.0) 1093 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:23:56.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:56 smithi145 bash[82329]: audit 2024-03-07T03:23:55.702470+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:56.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:56 smithi189 bash[66969]: cluster 2024-03-07T03:23:55.038398+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e665: 8 total, 8 up, 8 in 2024-03-07T03:23:56.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:56 smithi189 bash[66969]: audit 2024-03-07T03:23:55.106488+0000 mon.a (mon.0) 1090 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:56.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:56 smithi189 bash[66969]: audit 2024-03-07T03:23:55.116019+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:56.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:56 smithi189 bash[66969]: audit 2024-03-07T03:23:55.693586+0000 mon.a (mon.0) 1092 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:23:56.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:56 smithi189 bash[66969]: audit 2024-03-07T03:23:55.694933+0000 mon.a (mon.0) 1093 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:23:56.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:56 smithi189 bash[66969]: audit 2024-03-07T03:23:55.702470+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:23:57.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:57 smithi145 bash[79878]: cluster 2024-03-07T03:23:56.806457+0000 mgr.y (mgr.34107) 363 : cluster 0 pgmap v653: 425 pgs: 425 active+clean; 31 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 23 MiB/s wr, 304 op/s 2024-03-07T03:23:57.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:57 smithi145 bash[82329]: cluster 2024-03-07T03:23:56.806457+0000 mgr.y (mgr.34107) 363 : cluster 0 pgmap v653: 425 pgs: 425 active+clean; 31 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 23 MiB/s wr, 304 op/s 2024-03-07T03:23:57.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:57 smithi189 bash[66969]: cluster 2024-03-07T03:23:56.806457+0000 mgr.y (mgr.34107) 363 : cluster 0 pgmap v653: 425 pgs: 425 active+clean; 31 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 23 MiB/s wr, 304 op/s 2024-03-07T03:23:58.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:58 smithi145 bash[79878]: cluster 2024-03-07T03:23:57.414486+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e666: 8 total, 8 up, 8 in 2024-03-07T03:23:58.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:58 smithi145 bash[82329]: cluster 2024-03-07T03:23:57.414486+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e666: 8 total, 8 up, 8 in 2024-03-07T03:23:58.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:58 smithi189 bash[66969]: cluster 2024-03-07T03:23:57.414486+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e666: 8 total, 8 up, 8 in 2024-03-07T03:23:59.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:23:59 smithi145 bash[79878]: cluster 2024-03-07T03:23:58.808827+0000 mgr.y (mgr.34107) 364 : cluster 0 pgmap v655: 425 pgs: 425 active+clean; 31 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 896 KiB/s rd, 19 MiB/s wr, 294 op/s 2024-03-07T03:23:59.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:23:59 smithi145 bash[82329]: cluster 2024-03-07T03:23:58.808827+0000 mgr.y (mgr.34107) 364 : cluster 0 pgmap v655: 425 pgs: 425 active+clean; 31 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 896 KiB/s rd, 19 MiB/s wr, 294 op/s 2024-03-07T03:23:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:23:59 smithi189 bash[66969]: cluster 2024-03-07T03:23:58.808827+0000 mgr.y (mgr.34107) 364 : cluster 0 pgmap v655: 425 pgs: 425 active+clean; 31 MiB data, 461 MiB used, 715 GiB / 715 GiB avail; 896 KiB/s rd, 19 MiB/s wr, 294 op/s 2024-03-07T03:24:00.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:00 smithi145 bash[79878]: cluster 2024-03-07T03:23:59.432350+0000 mon.a (mon.0) 1096 : cluster 0 osdmap e667: 8 total, 8 up, 8 in 2024-03-07T03:24:00.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:00 smithi145 bash[82329]: cluster 2024-03-07T03:23:59.432350+0000 mon.a (mon.0) 1096 : cluster 0 osdmap e667: 8 total, 8 up, 8 in 2024-03-07T03:24:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:00 smithi189 bash[66969]: cluster 2024-03-07T03:23:59.432350+0000 mon.a (mon.0) 1096 : cluster 0 osdmap e667: 8 total, 8 up, 8 in 2024-03-07T03:24:01.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:01 smithi145 bash[79878]: cluster 2024-03-07T03:24:00.810405+0000 mgr.y (mgr.34107) 365 : cluster 0 pgmap v657: 425 pgs: 425 active+clean; 51 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 3.3 MiB/s wr, 174 op/s 2024-03-07T03:24:01.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:01 smithi145 bash[82329]: cluster 2024-03-07T03:24:00.810405+0000 mgr.y (mgr.34107) 365 : cluster 0 pgmap v657: 425 pgs: 425 active+clean; 51 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 3.3 MiB/s wr, 174 op/s 2024-03-07T03:24:01.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:01 smithi189 bash[66969]: cluster 2024-03-07T03:24:00.810405+0000 mgr.y (mgr.34107) 365 : cluster 0 pgmap v657: 425 pgs: 425 active+clean; 51 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 3.3 MiB/s wr, 174 op/s 2024-03-07T03:24:02.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:24:02 smithi189 bash[59814]: ts=2024-03-07T03:24:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:24:03.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:24:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:24:02] "GET /metrics HTTP/1.1" 200 60651 "" "Prometheus/2.43.0" 2024-03-07T03:24:03.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:03 smithi145 bash[79878]: cluster 2024-03-07T03:24:02.414158+0000 mon.a (mon.0) 1097 : cluster 0 osdmap e668: 8 total, 8 up, 8 in 2024-03-07T03:24:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:03 smithi145 bash[79878]: audit 2024-03-07T03:24:02.644638+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:03 smithi145 bash[79878]: cluster 2024-03-07T03:24:02.811117+0000 mgr.y (mgr.34107) 366 : cluster 0 pgmap v659: 425 pgs: 425 active+clean; 55 MiB data, 503 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 4.0 MiB/s wr, 70 op/s 2024-03-07T03:24:03.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:03 smithi145 bash[82329]: cluster 2024-03-07T03:24:02.414158+0000 mon.a (mon.0) 1097 : cluster 0 osdmap e668: 8 total, 8 up, 8 in 2024-03-07T03:24:03.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:03 smithi145 bash[82329]: audit 2024-03-07T03:24:02.644638+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:03.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:03 smithi145 bash[82329]: cluster 2024-03-07T03:24:02.811117+0000 mgr.y (mgr.34107) 366 : cluster 0 pgmap v659: 425 pgs: 425 active+clean; 55 MiB data, 503 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 4.0 MiB/s wr, 70 op/s 2024-03-07T03:24:03.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:03 smithi189 bash[66969]: cluster 2024-03-07T03:24:02.414158+0000 mon.a (mon.0) 1097 : cluster 0 osdmap e668: 8 total, 8 up, 8 in 2024-03-07T03:24:03.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:03 smithi189 bash[66969]: audit 2024-03-07T03:24:02.644638+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:03.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:03 smithi189 bash[66969]: cluster 2024-03-07T03:24:02.811117+0000 mgr.y (mgr.34107) 366 : cluster 0 pgmap v659: 425 pgs: 425 active+clean; 55 MiB data, 503 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 4.0 MiB/s wr, 70 op/s 2024-03-07T03:24:04.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:04 smithi145 bash[79878]: audit 2024-03-07T03:24:03.947710+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 4]}]: dispatch 2024-03-07T03:24:04.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:04 smithi145 bash[79878]: audit 2024-03-07T03:24:03.948026+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]}]: dispatch 2024-03-07T03:24:04.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:04 smithi145 bash[79878]: audit 2024-03-07T03:24:03.948277+0000 mon.a (mon.0) 1101 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.7", "id": [6, 0]}]: dispatch 2024-03-07T03:24:04.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:04 smithi145 bash[82329]: audit 2024-03-07T03:24:03.947710+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 4]}]: dispatch 2024-03-07T03:24:04.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:04 smithi145 bash[82329]: audit 2024-03-07T03:24:03.948026+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]}]: dispatch 2024-03-07T03:24:04.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:04 smithi145 bash[82329]: audit 2024-03-07T03:24:03.948277+0000 mon.a (mon.0) 1101 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.7", "id": [6, 0]}]: dispatch 2024-03-07T03:24:04.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:04 smithi189 bash[66969]: audit 2024-03-07T03:24:03.947710+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 4]}]: dispatch 2024-03-07T03:24:04.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:04 smithi189 bash[66969]: audit 2024-03-07T03:24:03.948026+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]}]: dispatch 2024-03-07T03:24:04.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:04 smithi189 bash[66969]: audit 2024-03-07T03:24:03.948277+0000 mon.a (mon.0) 1101 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.7", "id": [6, 0]}]: dispatch 2024-03-07T03:24:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:05 smithi145 bash[79878]: audit 2024-03-07T03:24:04.427905+0000 mon.a (mon.0) 1102 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 4]}]': finished 2024-03-07T03:24:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:05 smithi145 bash[79878]: audit 2024-03-07T03:24:04.427998+0000 mon.a (mon.0) 1103 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]}]': finished 2024-03-07T03:24:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:05 smithi145 bash[79878]: audit 2024-03-07T03:24:04.428062+0000 mon.a (mon.0) 1104 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.7", "id": [6, 0]}]': finished 2024-03-07T03:24:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:05 smithi145 bash[79878]: cluster 2024-03-07T03:24:04.433265+0000 mon.a (mon.0) 1105 : cluster 0 osdmap e669: 8 total, 8 up, 8 in 2024-03-07T03:24:05.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:05 smithi145 bash[79878]: cluster 2024-03-07T03:24:04.811782+0000 mgr.y (mgr.34107) 367 : cluster 0 pgmap v661: 425 pgs: 425 active+clean; 60 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 4.8 MiB/s wr, 54 op/s 2024-03-07T03:24:05.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:05 smithi145 bash[82329]: audit 2024-03-07T03:24:04.427905+0000 mon.a (mon.0) 1102 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 4]}]': finished 2024-03-07T03:24:05.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:05 smithi145 bash[82329]: audit 2024-03-07T03:24:04.427998+0000 mon.a (mon.0) 1103 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]}]': finished 2024-03-07T03:24:05.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:05 smithi145 bash[82329]: audit 2024-03-07T03:24:04.428062+0000 mon.a (mon.0) 1104 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.7", "id": [6, 0]}]': finished 2024-03-07T03:24:05.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:05 smithi145 bash[82329]: cluster 2024-03-07T03:24:04.433265+0000 mon.a (mon.0) 1105 : cluster 0 osdmap e669: 8 total, 8 up, 8 in 2024-03-07T03:24:05.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:05 smithi145 bash[82329]: cluster 2024-03-07T03:24:04.811782+0000 mgr.y (mgr.34107) 367 : cluster 0 pgmap v661: 425 pgs: 425 active+clean; 60 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 4.8 MiB/s wr, 54 op/s 2024-03-07T03:24:05.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:05 smithi189 bash[66969]: audit 2024-03-07T03:24:04.427905+0000 mon.a (mon.0) 1102 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.2", "id": [7, 4]}]': finished 2024-03-07T03:24:05.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:05 smithi189 bash[66969]: audit 2024-03-07T03:24:04.427998+0000 mon.a (mon.0) 1103 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 0]}]': finished 2024-03-07T03:24:05.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:05 smithi189 bash[66969]: audit 2024-03-07T03:24:04.428062+0000 mon.a (mon.0) 1104 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.7", "id": [6, 0]}]': finished 2024-03-07T03:24:05.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:05 smithi189 bash[66969]: cluster 2024-03-07T03:24:04.433265+0000 mon.a (mon.0) 1105 : cluster 0 osdmap e669: 8 total, 8 up, 8 in 2024-03-07T03:24:05.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:05 smithi189 bash[66969]: cluster 2024-03-07T03:24:04.811782+0000 mgr.y (mgr.34107) 367 : cluster 0 pgmap v661: 425 pgs: 425 active+clean; 60 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 4.8 MiB/s wr, 54 op/s 2024-03-07T03:24:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:06 smithi145 bash[79878]: cluster 2024-03-07T03:24:05.444399+0000 mon.a (mon.0) 1106 : cluster 0 osdmap e670: 8 total, 8 up, 8 in 2024-03-07T03:24:06.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:06 smithi145 bash[82329]: cluster 2024-03-07T03:24:05.444399+0000 mon.a (mon.0) 1106 : cluster 0 osdmap e670: 8 total, 8 up, 8 in 2024-03-07T03:24:06.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:06 smithi189 bash[66969]: cluster 2024-03-07T03:24:05.444399+0000 mon.a (mon.0) 1106 : cluster 0 osdmap e670: 8 total, 8 up, 8 in 2024-03-07T03:24:07.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:07 smithi145 bash[79878]: cluster 2024-03-07T03:24:06.813190+0000 mgr.y (mgr.34107) 368 : cluster 0 pgmap v663: 425 pgs: 2 peering, 423 active+clean; 60 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 1.5 MiB/s wr, 25 op/s 2024-03-07T03:24:07.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:07 smithi145 bash[82329]: cluster 2024-03-07T03:24:06.813190+0000 mgr.y (mgr.34107) 368 : cluster 0 pgmap v663: 425 pgs: 2 peering, 423 active+clean; 60 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 1.5 MiB/s wr, 25 op/s 2024-03-07T03:24:07.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:07 smithi189 bash[66969]: cluster 2024-03-07T03:24:06.813190+0000 mgr.y (mgr.34107) 368 : cluster 0 pgmap v663: 425 pgs: 2 peering, 423 active+clean; 60 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 1.5 MiB/s wr, 25 op/s 2024-03-07T03:24:08.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:08 smithi145 bash[79878]: cluster 2024-03-07T03:24:07.440805+0000 mon.a (mon.0) 1107 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:24:08.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:08 smithi145 bash[82329]: cluster 2024-03-07T03:24:07.440805+0000 mon.a (mon.0) 1107 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:24:08.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:08 smithi189 bash[66969]: cluster 2024-03-07T03:24:07.440805+0000 mon.a (mon.0) 1107 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:24:09.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:09 smithi145 bash[82329]: cluster 2024-03-07T03:24:08.814791+0000 mgr.y (mgr.34107) 369 : cluster 0 pgmap v664: 425 pgs: 2 peering, 423 active+clean; 64 MiB data, 556 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 1.4 MiB/s wr, 29 op/s 2024-03-07T03:24:09.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:09 smithi145 bash[79878]: cluster 2024-03-07T03:24:08.814791+0000 mgr.y (mgr.34107) 369 : cluster 0 pgmap v664: 425 pgs: 2 peering, 423 active+clean; 64 MiB data, 556 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 1.4 MiB/s wr, 29 op/s 2024-03-07T03:24:09.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:09 smithi189 bash[66969]: cluster 2024-03-07T03:24:08.814791+0000 mgr.y (mgr.34107) 369 : cluster 0 pgmap v664: 425 pgs: 2 peering, 423 active+clean; 64 MiB data, 556 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 1.4 MiB/s wr, 29 op/s 2024-03-07T03:24:11.248 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.LUKS1UnderLUKS2WithoutResize (16079 ms) 2024-03-07T03:24:11.248 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.LUKS2UnderLUKS1WithoutResize 2024-03-07T03:24:11.248 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:24:11.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:10 smithi145 bash[82329]: cluster 2024-03-07T03:24:10.815898+0000 mgr.y (mgr.34107) 370 : cluster 0 pgmap v665: 425 pgs: 425 active+clean; 68 MiB data, 581 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 1.6 MiB/s wr, 23 op/s 2024-03-07T03:24:11.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:10 smithi145 bash[82329]: cluster 2024-03-07T03:24:10.868757+0000 mon.a (mon.0) 1108 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-07T03:24:11.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:10 smithi145 bash[79878]: cluster 2024-03-07T03:24:10.815898+0000 mgr.y (mgr.34107) 370 : cluster 0 pgmap v665: 425 pgs: 425 active+clean; 68 MiB data, 581 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 1.6 MiB/s wr, 23 op/s 2024-03-07T03:24:11.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:10 smithi145 bash[79878]: cluster 2024-03-07T03:24:10.868757+0000 mon.a (mon.0) 1108 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-07T03:24:11.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:10 smithi189 bash[66969]: cluster 2024-03-07T03:24:10.815898+0000 mgr.y (mgr.34107) 370 : cluster 0 pgmap v665: 425 pgs: 425 active+clean; 68 MiB data, 581 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 1.6 MiB/s wr, 23 op/s 2024-03-07T03:24:11.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:10 smithi189 bash[66969]: cluster 2024-03-07T03:24:10.868757+0000 mon.a (mon.0) 1108 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-07T03:24:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:24:12 smithi189 bash[59814]: ts=2024-03-07T03:24:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:24:13.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:24:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:24:12] "GET /metrics HTTP/1.1" 200 60651 "" "Prometheus/2.43.0" 2024-03-07T03:24:14.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:13 smithi145 bash[79878]: cluster 2024-03-07T03:24:12.817207+0000 mgr.y (mgr.34107) 371 : cluster 0 pgmap v666: 425 pgs: 425 active+clean; 76 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 1.9 MiB/s wr, 8 op/s 2024-03-07T03:24:14.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:13 smithi145 bash[82329]: cluster 2024-03-07T03:24:12.817207+0000 mgr.y (mgr.34107) 371 : cluster 0 pgmap v666: 425 pgs: 425 active+clean; 76 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 1.9 MiB/s wr, 8 op/s 2024-03-07T03:24:14.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:13 smithi189 bash[66969]: cluster 2024-03-07T03:24:12.817207+0000 mgr.y (mgr.34107) 371 : cluster 0 pgmap v666: 425 pgs: 425 active+clean; 76 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 1.9 MiB/s wr, 8 op/s 2024-03-07T03:24:16.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:15 smithi145 bash[79878]: cluster 2024-03-07T03:24:14.818629+0000 mgr.y (mgr.34107) 372 : cluster 0 pgmap v667: 425 pgs: 425 active+clean; 76 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 472 KiB/s rd, 1.6 MiB/s wr, 7 op/s 2024-03-07T03:24:16.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:15 smithi145 bash[82329]: cluster 2024-03-07T03:24:14.818629+0000 mgr.y (mgr.34107) 372 : cluster 0 pgmap v667: 425 pgs: 425 active+clean; 76 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 472 KiB/s rd, 1.6 MiB/s wr, 7 op/s 2024-03-07T03:24:16.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:15 smithi189 bash[66969]: cluster 2024-03-07T03:24:14.818629+0000 mgr.y (mgr.34107) 372 : cluster 0 pgmap v667: 425 pgs: 425 active+clean; 76 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 472 KiB/s rd, 1.6 MiB/s wr, 7 op/s 2024-03-07T03:24:18.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:17 smithi145 bash[79878]: cluster 2024-03-07T03:24:16.820218+0000 mgr.y (mgr.34107) 373 : cluster 0 pgmap v668: 425 pgs: 425 active+clean; 76 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.4 MiB/s wr, 8 op/s 2024-03-07T03:24:18.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:17 smithi145 bash[79878]: audit 2024-03-07T03:24:17.644939+0000 mon.a (mon.0) 1109 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:18.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:17 smithi145 bash[82329]: cluster 2024-03-07T03:24:16.820218+0000 mgr.y (mgr.34107) 373 : cluster 0 pgmap v668: 425 pgs: 425 active+clean; 76 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.4 MiB/s wr, 8 op/s 2024-03-07T03:24:18.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:17 smithi145 bash[82329]: audit 2024-03-07T03:24:17.644939+0000 mon.a (mon.0) 1109 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:18.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:17 smithi189 bash[66969]: cluster 2024-03-07T03:24:16.820218+0000 mgr.y (mgr.34107) 373 : cluster 0 pgmap v668: 425 pgs: 425 active+clean; 76 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.4 MiB/s wr, 8 op/s 2024-03-07T03:24:18.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:17 smithi189 bash[66969]: audit 2024-03-07T03:24:17.644939+0000 mon.a (mon.0) 1109 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:20.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:19 smithi145 bash[79878]: cluster 2024-03-07T03:24:18.821756+0000 mgr.y (mgr.34107) 374 : cluster 0 pgmap v669: 425 pgs: 425 active+clean; 76 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-03-07T03:24:20.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:19 smithi145 bash[79878]: audit 2024-03-07T03:24:18.924017+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:24:20.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:19 smithi145 bash[82329]: cluster 2024-03-07T03:24:18.821756+0000 mgr.y (mgr.34107) 374 : cluster 0 pgmap v669: 425 pgs: 425 active+clean; 76 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-03-07T03:24:20.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:19 smithi145 bash[82329]: audit 2024-03-07T03:24:18.924017+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:24:20.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:19 smithi189 bash[66969]: cluster 2024-03-07T03:24:18.821756+0000 mgr.y (mgr.34107) 374 : cluster 0 pgmap v669: 425 pgs: 425 active+clean; 76 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-03-07T03:24:20.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:19 smithi189 bash[66969]: audit 2024-03-07T03:24:18.924017+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:24:21.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:20 smithi145 bash[79878]: cluster 2024-03-07T03:24:19.932351+0000 mon.a (mon.0) 1111 : cluster 0 osdmap e671: 8 total, 8 up, 8 in 2024-03-07T03:24:21.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:20 smithi145 bash[79878]: cluster 2024-03-07T03:24:20.823357+0000 mgr.y (mgr.34107) 375 : cluster 0 pgmap v671: 425 pgs: 425 active+clean; 92 MiB data, 647 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-03-07T03:24:21.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:20 smithi145 bash[82329]: cluster 2024-03-07T03:24:19.932351+0000 mon.a (mon.0) 1111 : cluster 0 osdmap e671: 8 total, 8 up, 8 in 2024-03-07T03:24:21.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:20 smithi145 bash[82329]: cluster 2024-03-07T03:24:20.823357+0000 mgr.y (mgr.34107) 375 : cluster 0 pgmap v671: 425 pgs: 425 active+clean; 92 MiB data, 647 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-03-07T03:24:21.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:20 smithi189 bash[66969]: cluster 2024-03-07T03:24:19.932351+0000 mon.a (mon.0) 1111 : cluster 0 osdmap e671: 8 total, 8 up, 8 in 2024-03-07T03:24:21.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:20 smithi189 bash[66969]: cluster 2024-03-07T03:24:20.823357+0000 mgr.y (mgr.34107) 375 : cluster 0 pgmap v671: 425 pgs: 425 active+clean; 92 MiB data, 647 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-03-07T03:24:22.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:24:22 smithi189 bash[59814]: ts=2024-03-07T03:24:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:24:23.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:24:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:24:22] "GET /metrics HTTP/1.1" 200 60650 "" "Prometheus/2.43.0" 2024-03-07T03:24:24.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:23 smithi145 bash[79878]: cluster 2024-03-07T03:24:22.825114+0000 mgr.y (mgr.34107) 376 : cluster 0 pgmap v672: 425 pgs: 425 active+clean; 112 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.6 MiB/s wr, 29 op/s 2024-03-07T03:24:24.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:23 smithi145 bash[82329]: cluster 2024-03-07T03:24:22.825114+0000 mgr.y (mgr.34107) 376 : cluster 0 pgmap v672: 425 pgs: 425 active+clean; 112 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.6 MiB/s wr, 29 op/s 2024-03-07T03:24:24.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:23 smithi189 bash[66969]: cluster 2024-03-07T03:24:22.825114+0000 mgr.y (mgr.34107) 376 : cluster 0 pgmap v672: 425 pgs: 425 active+clean; 112 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.6 MiB/s wr, 29 op/s 2024-03-07T03:24:26.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:25 smithi145 bash[79878]: cluster 2024-03-07T03:24:24.827138+0000 mgr.y (mgr.34107) 377 : cluster 0 pgmap v673: 425 pgs: 425 active+clean; 117 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 941 KiB/s rd, 4.1 MiB/s wr, 30 op/s 2024-03-07T03:24:26.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:25 smithi145 bash[82329]: cluster 2024-03-07T03:24:24.827138+0000 mgr.y (mgr.34107) 377 : cluster 0 pgmap v673: 425 pgs: 425 active+clean; 117 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 941 KiB/s rd, 4.1 MiB/s wr, 30 op/s 2024-03-07T03:24:26.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:25 smithi189 bash[66969]: cluster 2024-03-07T03:24:24.827138+0000 mgr.y (mgr.34107) 377 : cluster 0 pgmap v673: 425 pgs: 425 active+clean; 117 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 941 KiB/s rd, 4.1 MiB/s wr, 30 op/s 2024-03-07T03:24:27.833 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.LUKS2UnderLUKS1WithoutResize (16585 ms) 2024-03-07T03:24:27.833 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.EncryptionFormatNoData 2024-03-07T03:24:27.833 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:24:28.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:27 smithi145 bash[79878]: cluster 2024-03-07T03:24:26.828663+0000 mgr.y (mgr.34107) 378 : cluster 0 pgmap v674: 425 pgs: 425 active+clean; 117 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:24:28.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:27 smithi145 bash[82329]: cluster 2024-03-07T03:24:26.828663+0000 mgr.y (mgr.34107) 378 : cluster 0 pgmap v674: 425 pgs: 425 active+clean; 117 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:24:28.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:27 smithi189 bash[66969]: cluster 2024-03-07T03:24:26.828663+0000 mgr.y (mgr.34107) 378 : cluster 0 pgmap v674: 425 pgs: 425 active+clean; 117 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:24:30.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:29 smithi145 bash[79878]: cluster 2024-03-07T03:24:28.830908+0000 mgr.y (mgr.34107) 379 : cluster 0 pgmap v675: 425 pgs: 425 active+clean; 121 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 190 KiB/s rd, 4.5 MiB/s wr, 43 op/s 2024-03-07T03:24:30.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:29 smithi145 bash[82329]: cluster 2024-03-07T03:24:28.830908+0000 mgr.y (mgr.34107) 379 : cluster 0 pgmap v675: 425 pgs: 425 active+clean; 121 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 190 KiB/s rd, 4.5 MiB/s wr, 43 op/s 2024-03-07T03:24:30.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:29 smithi189 bash[66969]: cluster 2024-03-07T03:24:28.830908+0000 mgr.y (mgr.34107) 379 : cluster 0 pgmap v675: 425 pgs: 425 active+clean; 121 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 190 KiB/s rd, 4.5 MiB/s wr, 43 op/s 2024-03-07T03:24:32.046 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.EncryptionFormatNoData (4213 ms) 2024-03-07T03:24:32.046 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.EncryptionLoadBadSize 2024-03-07T03:24:32.046 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:24:32.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:31 smithi189 bash[66969]: cluster 2024-03-07T03:24:30.832846+0000 mgr.y (mgr.34107) 380 : cluster 0 pgmap v676: 425 pgs: 425 active+clean; 121 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-03-07T03:24:32.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:31 smithi145 bash[82329]: cluster 2024-03-07T03:24:30.832846+0000 mgr.y (mgr.34107) 380 : cluster 0 pgmap v676: 425 pgs: 425 active+clean; 121 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-03-07T03:24:32.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:31 smithi145 bash[79878]: cluster 2024-03-07T03:24:30.832846+0000 mgr.y (mgr.34107) 380 : cluster 0 pgmap v676: 425 pgs: 425 active+clean; 121 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-03-07T03:24:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:24:32 smithi189 bash[59814]: ts=2024-03-07T03:24:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:24:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:32 smithi145 bash[82329]: audit 2024-03-07T03:24:32.646136+0000 mon.a (mon.0) 1112 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:32 smithi145 bash[79878]: audit 2024-03-07T03:24:32.646136+0000 mon.a (mon.0) 1112 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:33.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:24:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:24:32] "GET /metrics HTTP/1.1" 200 60647 "" "Prometheus/2.43.0" 2024-03-07T03:24:33.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:32 smithi189 bash[66969]: audit 2024-03-07T03:24:32.646136+0000 mon.a (mon.0) 1112 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:34.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:33 smithi145 bash[79878]: cluster 2024-03-07T03:24:32.834655+0000 mgr.y (mgr.34107) 381 : cluster 0 pgmap v677: 425 pgs: 425 active+clean; 121 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 28 op/s 2024-03-07T03:24:34.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:33 smithi145 bash[82329]: cluster 2024-03-07T03:24:32.834655+0000 mgr.y (mgr.34107) 381 : cluster 0 pgmap v677: 425 pgs: 425 active+clean; 121 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 28 op/s 2024-03-07T03:24:34.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:33 smithi189 bash[66969]: cluster 2024-03-07T03:24:32.834655+0000 mgr.y (mgr.34107) 381 : cluster 0 pgmap v677: 425 pgs: 425 active+clean; 121 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 28 op/s 2024-03-07T03:24:35.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:34 smithi145 bash[79878]: cluster 2024-03-07T03:24:34.836786+0000 mgr.y (mgr.34107) 382 : cluster 0 pgmap v678: 425 pgs: 425 active+clean; 121 MiB data, 736 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 769 KiB/s wr, 19 op/s 2024-03-07T03:24:35.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:34 smithi145 bash[82329]: cluster 2024-03-07T03:24:34.836786+0000 mgr.y (mgr.34107) 382 : cluster 0 pgmap v678: 425 pgs: 425 active+clean; 121 MiB data, 736 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 769 KiB/s wr, 19 op/s 2024-03-07T03:24:35.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:34 smithi189 bash[66969]: cluster 2024-03-07T03:24:34.836786+0000 mgr.y (mgr.34107) 382 : cluster 0 pgmap v678: 425 pgs: 425 active+clean; 121 MiB data, 736 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 769 KiB/s wr, 19 op/s 2024-03-07T03:24:37.715 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.EncryptionLoadBadSize (5669 ms) 2024-03-07T03:24:37.715 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.EncryptionLoadBadStripePattern 2024-03-07T03:24:37.716 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_librbd.cc:2840: Skipped 2024-03-07T03:24:37.716 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:24:37.716 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestLibRBD.EncryptionLoadBadStripePattern (0 ms) 2024-03-07T03:24:37.716 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.EncryptionLoadFormatMismatch 2024-03-07T03:24:37.716 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:24:38.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:37 smithi145 bash[82329]: cluster 2024-03-07T03:24:36.839500+0000 mgr.y (mgr.34107) 383 : cluster 0 pgmap v679: 425 pgs: 425 active+clean; 125 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 684 KiB/s wr, 20 op/s 2024-03-07T03:24:38.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:37 smithi145 bash[79878]: cluster 2024-03-07T03:24:36.839500+0000 mgr.y (mgr.34107) 383 : cluster 0 pgmap v679: 425 pgs: 425 active+clean; 125 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 684 KiB/s wr, 20 op/s 2024-03-07T03:24:38.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:37 smithi189 bash[66969]: cluster 2024-03-07T03:24:36.839500+0000 mgr.y (mgr.34107) 383 : cluster 0 pgmap v679: 425 pgs: 425 active+clean; 125 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 684 KiB/s wr, 20 op/s 2024-03-07T03:24:40.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:39 smithi145 bash[79878]: cluster 2024-03-07T03:24:38.841493+0000 mgr.y (mgr.34107) 384 : cluster 0 pgmap v680: 425 pgs: 425 active+clean; 129 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.0 MiB/s wr, 23 op/s 2024-03-07T03:24:40.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:39 smithi145 bash[82329]: cluster 2024-03-07T03:24:38.841493+0000 mgr.y (mgr.34107) 384 : cluster 0 pgmap v680: 425 pgs: 425 active+clean; 129 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.0 MiB/s wr, 23 op/s 2024-03-07T03:24:40.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:39 smithi189 bash[66969]: cluster 2024-03-07T03:24:38.841493+0000 mgr.y (mgr.34107) 384 : cluster 0 pgmap v680: 425 pgs: 425 active+clean; 129 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.0 MiB/s wr, 23 op/s 2024-03-07T03:24:41.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:40 smithi145 bash[82329]: cluster 2024-03-07T03:24:40.843549+0000 mgr.y (mgr.34107) 385 : cluster 0 pgmap v681: 425 pgs: 425 active+clean; 129 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 683 KiB/s wr, 19 op/s 2024-03-07T03:24:41.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:40 smithi145 bash[79878]: cluster 2024-03-07T03:24:40.843549+0000 mgr.y (mgr.34107) 385 : cluster 0 pgmap v681: 425 pgs: 425 active+clean; 129 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 683 KiB/s wr, 19 op/s 2024-03-07T03:24:41.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:40 smithi189 bash[66969]: cluster 2024-03-07T03:24:40.843549+0000 mgr.y (mgr.34107) 385 : cluster 0 pgmap v681: 425 pgs: 425 active+clean; 129 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 683 KiB/s wr, 19 op/s 2024-03-07T03:24:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:24:42 smithi189 bash[59814]: ts=2024-03-07T03:24:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:24:43.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:24:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:24:42] "GET /metrics HTTP/1.1" 200 60647 "" "Prometheus/2.43.0" 2024-03-07T03:24:43.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:42 smithi145 bash[79878]: cluster 2024-03-07T03:24:41.964907+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e672: 8 total, 8 up, 8 in 2024-03-07T03:24:43.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:42 smithi145 bash[79878]: cluster 2024-03-07T03:24:42.844548+0000 mgr.y (mgr.34107) 386 : cluster 0 pgmap v683: 425 pgs: 425 active+clean; 129 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 820 KiB/s wr, 22 op/s 2024-03-07T03:24:43.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:42 smithi145 bash[82329]: cluster 2024-03-07T03:24:41.964907+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e672: 8 total, 8 up, 8 in 2024-03-07T03:24:43.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:42 smithi145 bash[82329]: cluster 2024-03-07T03:24:42.844548+0000 mgr.y (mgr.34107) 386 : cluster 0 pgmap v683: 425 pgs: 425 active+clean; 129 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 820 KiB/s wr, 22 op/s 2024-03-07T03:24:43.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:42 smithi189 bash[66969]: cluster 2024-03-07T03:24:41.964907+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e672: 8 total, 8 up, 8 in 2024-03-07T03:24:43.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:42 smithi189 bash[66969]: cluster 2024-03-07T03:24:42.844548+0000 mgr.y (mgr.34107) 386 : cluster 0 pgmap v683: 425 pgs: 425 active+clean; 129 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 820 KiB/s wr, 22 op/s 2024-03-07T03:24:44.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:43 smithi145 bash[79878]: cluster 2024-03-07T03:24:42.974684+0000 mon.a (mon.0) 1114 : cluster 0 osdmap e673: 8 total, 8 up, 8 in 2024-03-07T03:24:44.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:43 smithi145 bash[82329]: cluster 2024-03-07T03:24:42.974684+0000 mon.a (mon.0) 1114 : cluster 0 osdmap e673: 8 total, 8 up, 8 in 2024-03-07T03:24:44.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:43 smithi189 bash[66969]: cluster 2024-03-07T03:24:42.974684+0000 mon.a (mon.0) 1114 : cluster 0 osdmap e673: 8 total, 8 up, 8 in 2024-03-07T03:24:45.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:44 smithi145 bash[79878]: cluster 2024-03-07T03:24:44.846126+0000 mgr.y (mgr.34107) 387 : cluster 0 pgmap v685: 425 pgs: 425 active+clean; 129 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 513 KiB/s wr, 19 op/s 2024-03-07T03:24:45.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:44 smithi145 bash[82329]: cluster 2024-03-07T03:24:44.846126+0000 mgr.y (mgr.34107) 387 : cluster 0 pgmap v685: 425 pgs: 425 active+clean; 129 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 513 KiB/s wr, 19 op/s 2024-03-07T03:24:45.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:44 smithi189 bash[66969]: cluster 2024-03-07T03:24:44.846126+0000 mgr.y (mgr.34107) 387 : cluster 0 pgmap v685: 425 pgs: 425 active+clean; 129 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 513 KiB/s wr, 19 op/s 2024-03-07T03:24:48.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:47 smithi145 bash[79878]: cluster 2024-03-07T03:24:46.847817+0000 mgr.y (mgr.34107) 388 : cluster 0 pgmap v686: 425 pgs: 425 active+clean; 133 MiB data, 764 MiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 515 KiB/s wr, 74 op/s 2024-03-07T03:24:48.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:47 smithi145 bash[79878]: audit 2024-03-07T03:24:47.645686+0000 mon.a (mon.0) 1115 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:48.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:47 smithi145 bash[82329]: cluster 2024-03-07T03:24:46.847817+0000 mgr.y (mgr.34107) 388 : cluster 0 pgmap v686: 425 pgs: 425 active+clean; 133 MiB data, 764 MiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 515 KiB/s wr, 74 op/s 2024-03-07T03:24:48.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:47 smithi145 bash[82329]: audit 2024-03-07T03:24:47.645686+0000 mon.a (mon.0) 1115 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:48.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:47 smithi189 bash[66969]: cluster 2024-03-07T03:24:46.847817+0000 mgr.y (mgr.34107) 388 : cluster 0 pgmap v686: 425 pgs: 425 active+clean; 133 MiB data, 764 MiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 515 KiB/s wr, 74 op/s 2024-03-07T03:24:48.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:47 smithi189 bash[66969]: audit 2024-03-07T03:24:47.645686+0000 mon.a (mon.0) 1115 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:49.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:48 smithi145 bash[79878]: cluster 2024-03-07T03:24:48.849134+0000 mgr.y (mgr.34107) 389 : cluster 0 pgmap v687: 425 pgs: 425 active+clean; 133 MiB data, 764 MiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 515 KiB/s wr, 68 op/s 2024-03-07T03:24:49.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:48 smithi145 bash[82329]: cluster 2024-03-07T03:24:48.849134+0000 mgr.y (mgr.34107) 389 : cluster 0 pgmap v687: 425 pgs: 425 active+clean; 133 MiB data, 764 MiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 515 KiB/s wr, 68 op/s 2024-03-07T03:24:49.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:48 smithi189 bash[66969]: cluster 2024-03-07T03:24:48.849134+0000 mgr.y (mgr.34107) 389 : cluster 0 pgmap v687: 425 pgs: 425 active+clean; 133 MiB data, 764 MiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 515 KiB/s wr, 68 op/s 2024-03-07T03:24:52.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:51 smithi189 bash[66969]: cluster 2024-03-07T03:24:50.850539+0000 mgr.y (mgr.34107) 390 : cluster 0 pgmap v688: 425 pgs: 425 active+clean; 133 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 463 KiB/s wr, 68 op/s 2024-03-07T03:24:52.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:51 smithi145 bash[79878]: cluster 2024-03-07T03:24:50.850539+0000 mgr.y (mgr.34107) 390 : cluster 0 pgmap v688: 425 pgs: 425 active+clean; 133 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 463 KiB/s wr, 68 op/s 2024-03-07T03:24:52.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:51 smithi145 bash[82329]: cluster 2024-03-07T03:24:50.850539+0000 mgr.y (mgr.34107) 390 : cluster 0 pgmap v688: 425 pgs: 425 active+clean; 133 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 463 KiB/s wr, 68 op/s 2024-03-07T03:24:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:24:52 smithi189 bash[59814]: ts=2024-03-07T03:24:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:24:53.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:24:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:24:52] "GET /metrics HTTP/1.1" 200 60646 "" "Prometheus/2.43.0" 2024-03-07T03:24:53.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:53 smithi145 bash[79878]: cluster 2024-03-07T03:24:52.851702+0000 mgr.y (mgr.34107) 391 : cluster 0 pgmap v689: 425 pgs: 425 active+clean; 141 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 1.2 MiB/s wr, 64 op/s 2024-03-07T03:24:53.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:53 smithi145 bash[82329]: cluster 2024-03-07T03:24:52.851702+0000 mgr.y (mgr.34107) 391 : cluster 0 pgmap v689: 425 pgs: 425 active+clean; 141 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 1.2 MiB/s wr, 64 op/s 2024-03-07T03:24:53.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:53 smithi189 bash[66969]: cluster 2024-03-07T03:24:52.851702+0000 mgr.y (mgr.34107) 391 : cluster 0 pgmap v689: 425 pgs: 425 active+clean; 141 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 1.2 MiB/s wr, 64 op/s 2024-03-07T03:24:55.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:54 smithi145 bash[79878]: cluster 2024-03-07T03:24:54.853016+0000 mgr.y (mgr.34107) 392 : cluster 0 pgmap v690: 425 pgs: 425 active+clean; 149 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.7 MiB/s wr, 55 op/s 2024-03-07T03:24:55.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:54 smithi145 bash[82329]: cluster 2024-03-07T03:24:54.853016+0000 mgr.y (mgr.34107) 392 : cluster 0 pgmap v690: 425 pgs: 425 active+clean; 149 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.7 MiB/s wr, 55 op/s 2024-03-07T03:24:55.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:54 smithi189 bash[66969]: cluster 2024-03-07T03:24:54.853016+0000 mgr.y (mgr.34107) 392 : cluster 0 pgmap v690: 425 pgs: 425 active+clean; 149 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.7 MiB/s wr, 55 op/s 2024-03-07T03:24:56.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:55 smithi145 bash[79878]: audit 2024-03-07T03:24:55.787135+0000 mon.a (mon.0) 1116 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:24:56.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:55 smithi145 bash[82329]: audit 2024-03-07T03:24:55.787135+0000 mon.a (mon.0) 1116 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:24:56.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:55 smithi189 bash[66969]: audit 2024-03-07T03:24:55.787135+0000 mon.a (mon.0) 1116 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:24:57.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:56 smithi145 bash[79878]: cluster 2024-03-07T03:24:56.854777+0000 mgr.y (mgr.34107) 393 : cluster 0 pgmap v691: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.7 MiB/s wr, 56 op/s 2024-03-07T03:24:57.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:56 smithi145 bash[82329]: cluster 2024-03-07T03:24:56.854777+0000 mgr.y (mgr.34107) 393 : cluster 0 pgmap v691: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.7 MiB/s wr, 56 op/s 2024-03-07T03:24:57.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:56 smithi189 bash[66969]: cluster 2024-03-07T03:24:56.854777+0000 mgr.y (mgr.34107) 393 : cluster 0 pgmap v691: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.7 MiB/s wr, 56 op/s 2024-03-07T03:24:59.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:24:58 smithi145 bash[79878]: cluster 2024-03-07T03:24:58.855906+0000 mgr.y (mgr.34107) 394 : cluster 0 pgmap v692: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:24:59.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:24:58 smithi145 bash[82329]: cluster 2024-03-07T03:24:58.855906+0000 mgr.y (mgr.34107) 394 : cluster 0 pgmap v692: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:24:59.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:24:58 smithi189 bash[66969]: cluster 2024-03-07T03:24:58.855906+0000 mgr.y (mgr.34107) 394 : cluster 0 pgmap v692: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:25:01.245 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:00 smithi145 bash[82329]: cluster 2024-03-07T03:25:00.856939+0000 mgr.y (mgr.34107) 395 : cluster 0 pgmap v693: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:25:01.245 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:00 smithi145 bash[79878]: cluster 2024-03-07T03:25:00.856939+0000 mgr.y (mgr.34107) 395 : cluster 0 pgmap v693: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:25:01.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:00 smithi189 bash[66969]: cluster 2024-03-07T03:25:00.856939+0000 mgr.y (mgr.34107) 395 : cluster 0 pgmap v693: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:25:02.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:25:02 smithi189 bash[59814]: ts=2024-03-07T03:25:02.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:25:03.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:25:02] "GET /metrics HTTP/1.1" 200 60648 "" "Prometheus/2.43.0" 2024-03-07T03:25:03.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[79878]: audit 2024-03-07T03:25:01.724825+0000 mon.a (mon.0) 1117 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[79878]: audit 2024-03-07T03:25:01.732035+0000 mon.a (mon.0) 1118 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[79878]: audit 2024-03-07T03:25:01.837187+0000 mon.a (mon.0) 1119 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[79878]: audit 2024-03-07T03:25:01.845016+0000 mon.a (mon.0) 1120 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[79878]: audit 2024-03-07T03:25:02.423823+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:25:03.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[79878]: audit 2024-03-07T03:25:02.424858+0000 mon.a (mon.0) 1122 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:25:03.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[79878]: audit 2024-03-07T03:25:02.430655+0000 mon.a (mon.0) 1123 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[79878]: audit 2024-03-07T03:25:02.646031+0000 mon.a (mon.0) 1124 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[82329]: audit 2024-03-07T03:25:01.724825+0000 mon.a (mon.0) 1117 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[82329]: audit 2024-03-07T03:25:01.732035+0000 mon.a (mon.0) 1118 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[82329]: audit 2024-03-07T03:25:01.837187+0000 mon.a (mon.0) 1119 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[82329]: audit 2024-03-07T03:25:01.845016+0000 mon.a (mon.0) 1120 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[82329]: audit 2024-03-07T03:25:02.423823+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:25:03.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[82329]: audit 2024-03-07T03:25:02.424858+0000 mon.a (mon.0) 1122 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:25:03.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[82329]: audit 2024-03-07T03:25:02.430655+0000 mon.a (mon.0) 1123 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:02 smithi145 bash[82329]: audit 2024-03-07T03:25:02.646031+0000 mon.a (mon.0) 1124 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:02 smithi189 bash[66969]: audit 2024-03-07T03:25:01.724825+0000 mon.a (mon.0) 1117 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:02 smithi189 bash[66969]: audit 2024-03-07T03:25:01.732035+0000 mon.a (mon.0) 1118 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:02 smithi189 bash[66969]: audit 2024-03-07T03:25:01.837187+0000 mon.a (mon.0) 1119 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:02 smithi189 bash[66969]: audit 2024-03-07T03:25:01.845016+0000 mon.a (mon.0) 1120 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:02 smithi189 bash[66969]: audit 2024-03-07T03:25:02.423823+0000 mon.a (mon.0) 1121 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:25:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:02 smithi189 bash[66969]: audit 2024-03-07T03:25:02.424858+0000 mon.a (mon.0) 1122 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:25:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:02 smithi189 bash[66969]: audit 2024-03-07T03:25:02.430655+0000 mon.a (mon.0) 1123 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:25:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:02 smithi189 bash[66969]: audit 2024-03-07T03:25:02.646031+0000 mon.a (mon.0) 1124 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:03 smithi145 bash[79878]: cluster 2024-03-07T03:25:02.858258+0000 mgr.y (mgr.34107) 396 : cluster 0 pgmap v694: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-03-07T03:25:04.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:03 smithi145 bash[82329]: cluster 2024-03-07T03:25:02.858258+0000 mgr.y (mgr.34107) 396 : cluster 0 pgmap v694: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-03-07T03:25:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:03 smithi189 bash[66969]: cluster 2024-03-07T03:25:02.858258+0000 mgr.y (mgr.34107) 396 : cluster 0 pgmap v694: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-03-07T03:25:05.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:04 smithi145 bash[79878]: cluster 2024-03-07T03:25:04.859616+0000 mgr.y (mgr.34107) 397 : cluster 0 pgmap v695: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 683 KiB/s wr, 7 op/s 2024-03-07T03:25:05.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:04 smithi145 bash[82329]: cluster 2024-03-07T03:25:04.859616+0000 mgr.y (mgr.34107) 397 : cluster 0 pgmap v695: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 683 KiB/s wr, 7 op/s 2024-03-07T03:25:05.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:04 smithi189 bash[66969]: cluster 2024-03-07T03:25:04.859616+0000 mgr.y (mgr.34107) 397 : cluster 0 pgmap v695: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 683 KiB/s wr, 7 op/s 2024-03-07T03:25:07.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:06 smithi145 bash[79878]: cluster 2024-03-07T03:25:06.861357+0000 mgr.y (mgr.34107) 398 : cluster 0 pgmap v696: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 682 B/s wr, 6 op/s 2024-03-07T03:25:07.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:06 smithi145 bash[82329]: cluster 2024-03-07T03:25:06.861357+0000 mgr.y (mgr.34107) 398 : cluster 0 pgmap v696: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 682 B/s wr, 6 op/s 2024-03-07T03:25:07.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:06 smithi189 bash[66969]: cluster 2024-03-07T03:25:06.861357+0000 mgr.y (mgr.34107) 398 : cluster 0 pgmap v696: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 682 B/s wr, 6 op/s 2024-03-07T03:25:09.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:08 smithi145 bash[82329]: cluster 2024-03-07T03:25:08.863299+0000 mgr.y (mgr.34107) 399 : cluster 0 pgmap v697: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 2 op/s 2024-03-07T03:25:09.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:08 smithi145 bash[79878]: cluster 2024-03-07T03:25:08.863299+0000 mgr.y (mgr.34107) 399 : cluster 0 pgmap v697: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 2 op/s 2024-03-07T03:25:09.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:08 smithi189 bash[66969]: cluster 2024-03-07T03:25:08.863299+0000 mgr.y (mgr.34107) 399 : cluster 0 pgmap v697: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 2 op/s 2024-03-07T03:25:11.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:10 smithi145 bash[79878]: cluster 2024-03-07T03:25:10.864690+0000 mgr.y (mgr.34107) 400 : cluster 0 pgmap v698: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1 op/s 2024-03-07T03:25:11.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:10 smithi145 bash[82329]: cluster 2024-03-07T03:25:10.864690+0000 mgr.y (mgr.34107) 400 : cluster 0 pgmap v698: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1 op/s 2024-03-07T03:25:11.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:10 smithi189 bash[66969]: cluster 2024-03-07T03:25:10.864690+0000 mgr.y (mgr.34107) 400 : cluster 0 pgmap v698: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1 op/s 2024-03-07T03:25:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:25:12 smithi189 bash[59814]: ts=2024-03-07T03:25:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:25:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:12 smithi145 bash[82329]: cluster 2024-03-07T03:25:12.865687+0000 mgr.y (mgr.34107) 401 : cluster 0 pgmap v699: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:25:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:12 smithi145 bash[79878]: cluster 2024-03-07T03:25:12.865687+0000 mgr.y (mgr.34107) 401 : cluster 0 pgmap v699: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:25:13.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:25:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:25:12] "GET /metrics HTTP/1.1" 200 60648 "" "Prometheus/2.43.0" 2024-03-07T03:25:13.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:12 smithi189 bash[66969]: cluster 2024-03-07T03:25:12.865687+0000 mgr.y (mgr.34107) 401 : cluster 0 pgmap v699: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:25:15.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:14 smithi145 bash[79878]: cluster 2024-03-07T03:25:14.867033+0000 mgr.y (mgr.34107) 402 : cluster 0 pgmap v700: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1 op/s 2024-03-07T03:25:15.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:14 smithi145 bash[82329]: cluster 2024-03-07T03:25:14.867033+0000 mgr.y (mgr.34107) 402 : cluster 0 pgmap v700: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1 op/s 2024-03-07T03:25:15.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:14 smithi189 bash[66969]: cluster 2024-03-07T03:25:14.867033+0000 mgr.y (mgr.34107) 402 : cluster 0 pgmap v700: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1 op/s 2024-03-07T03:25:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:16 smithi145 bash[79878]: cluster 2024-03-07T03:25:16.868777+0000 mgr.y (mgr.34107) 403 : cluster 0 pgmap v701: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1 op/s 2024-03-07T03:25:17.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:16 smithi145 bash[82329]: cluster 2024-03-07T03:25:16.868777+0000 mgr.y (mgr.34107) 403 : cluster 0 pgmap v701: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1 op/s 2024-03-07T03:25:17.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:16 smithi189 bash[66969]: cluster 2024-03-07T03:25:16.868777+0000 mgr.y (mgr.34107) 403 : cluster 0 pgmap v701: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1 op/s 2024-03-07T03:25:18.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:17 smithi145 bash[79878]: audit 2024-03-07T03:25:17.646331+0000 mon.a (mon.0) 1125 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:18.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:17 smithi145 bash[82329]: audit 2024-03-07T03:25:17.646331+0000 mon.a (mon.0) 1125 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:18.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:17 smithi189 bash[66969]: audit 2024-03-07T03:25:17.646331+0000 mon.a (mon.0) 1125 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:19.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:18 smithi145 bash[79878]: cluster 2024-03-07T03:25:18.870139+0000 mgr.y (mgr.34107) 404 : cluster 0 pgmap v702: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1 op/s 2024-03-07T03:25:19.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:18 smithi145 bash[82329]: cluster 2024-03-07T03:25:18.870139+0000 mgr.y (mgr.34107) 404 : cluster 0 pgmap v702: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1 op/s 2024-03-07T03:25:19.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:18 smithi189 bash[66969]: cluster 2024-03-07T03:25:18.870139+0000 mgr.y (mgr.34107) 404 : cluster 0 pgmap v702: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1 op/s 2024-03-07T03:25:21.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:20 smithi145 bash[79878]: cluster 2024-03-07T03:25:20.871061+0000 mgr.y (mgr.34107) 405 : cluster 0 pgmap v703: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1 op/s 2024-03-07T03:25:21.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:20 smithi145 bash[82329]: cluster 2024-03-07T03:25:20.871061+0000 mgr.y (mgr.34107) 405 : cluster 0 pgmap v703: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1 op/s 2024-03-07T03:25:21.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:20 smithi189 bash[66969]: cluster 2024-03-07T03:25:20.871061+0000 mgr.y (mgr.34107) 405 : cluster 0 pgmap v703: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1 op/s 2024-03-07T03:25:22.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:25:22 smithi189 bash[59814]: ts=2024-03-07T03:25:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:25:23.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:22 smithi145 bash[79878]: cluster 2024-03-07T03:25:22.872714+0000 mgr.y (mgr.34107) 406 : cluster 0 pgmap v704: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:25:23.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:22 smithi145 bash[82329]: cluster 2024-03-07T03:25:22.872714+0000 mgr.y (mgr.34107) 406 : cluster 0 pgmap v704: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:25:23.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:25:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:25:22] "GET /metrics HTTP/1.1" 200 60645 "" "Prometheus/2.43.0" 2024-03-07T03:25:23.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:22 smithi189 bash[66969]: cluster 2024-03-07T03:25:22.872714+0000 mgr.y (mgr.34107) 406 : cluster 0 pgmap v704: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:25:25.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:24 smithi145 bash[82329]: cluster 2024-03-07T03:25:24.874827+0000 mgr.y (mgr.34107) 407 : cluster 0 pgmap v705: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1 op/s 2024-03-07T03:25:25.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:24 smithi145 bash[79878]: cluster 2024-03-07T03:25:24.874827+0000 mgr.y (mgr.34107) 407 : cluster 0 pgmap v705: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1 op/s 2024-03-07T03:25:25.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:24 smithi189 bash[66969]: cluster 2024-03-07T03:25:24.874827+0000 mgr.y (mgr.34107) 407 : cluster 0 pgmap v705: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1 op/s 2024-03-07T03:25:27.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:26 smithi145 bash[79878]: cluster 2024-03-07T03:25:26.877499+0000 mgr.y (mgr.34107) 408 : cluster 0 pgmap v706: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-03-07T03:25:27.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:26 smithi145 bash[82329]: cluster 2024-03-07T03:25:26.877499+0000 mgr.y (mgr.34107) 408 : cluster 0 pgmap v706: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-03-07T03:25:27.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:26 smithi189 bash[66969]: cluster 2024-03-07T03:25:26.877499+0000 mgr.y (mgr.34107) 408 : cluster 0 pgmap v706: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-03-07T03:25:29.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:28 smithi145 bash[79878]: cluster 2024-03-07T03:25:28.879431+0000 mgr.y (mgr.34107) 409 : cluster 0 pgmap v707: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 0 op/s 2024-03-07T03:25:29.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:28 smithi145 bash[82329]: cluster 2024-03-07T03:25:28.879431+0000 mgr.y (mgr.34107) 409 : cluster 0 pgmap v707: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 0 op/s 2024-03-07T03:25:29.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:28 smithi189 bash[66969]: cluster 2024-03-07T03:25:28.879431+0000 mgr.y (mgr.34107) 409 : cluster 0 pgmap v707: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 0 op/s 2024-03-07T03:25:31.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:30 smithi145 bash[82329]: cluster 2024-03-07T03:25:30.881350+0000 mgr.y (mgr.34107) 410 : cluster 0 pgmap v708: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 6 op/s 2024-03-07T03:25:31.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:30 smithi145 bash[79878]: cluster 2024-03-07T03:25:30.881350+0000 mgr.y (mgr.34107) 410 : cluster 0 pgmap v708: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 6 op/s 2024-03-07T03:25:31.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:30 smithi189 bash[66969]: cluster 2024-03-07T03:25:30.881350+0000 mgr.y (mgr.34107) 410 : cluster 0 pgmap v708: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 6 op/s 2024-03-07T03:25:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:25:32 smithi189 bash[59814]: ts=2024-03-07T03:25:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:25:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:32 smithi145 bash[79878]: audit 2024-03-07T03:25:32.647417+0000 mon.a (mon.0) 1126 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:33.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:25:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:25:32] "GET /metrics HTTP/1.1" 200 60646 "" "Prometheus/2.43.0" 2024-03-07T03:25:33.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:32 smithi145 bash[82329]: audit 2024-03-07T03:25:32.647417+0000 mon.a (mon.0) 1126 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:33.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:32 smithi189 bash[66969]: audit 2024-03-07T03:25:32.647417+0000 mon.a (mon.0) 1126 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:34.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:33 smithi145 bash[79878]: cluster 2024-03-07T03:25:32.883017+0000 mgr.y (mgr.34107) 411 : cluster 0 pgmap v709: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 492 KiB/s rd, 8 op/s 2024-03-07T03:25:34.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:33 smithi145 bash[82329]: cluster 2024-03-07T03:25:32.883017+0000 mgr.y (mgr.34107) 411 : cluster 0 pgmap v709: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 492 KiB/s rd, 8 op/s 2024-03-07T03:25:34.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:33 smithi189 bash[66969]: cluster 2024-03-07T03:25:32.883017+0000 mgr.y (mgr.34107) 411 : cluster 0 pgmap v709: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 492 KiB/s rd, 8 op/s 2024-03-07T03:25:35.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:34 smithi145 bash[79878]: cluster 2024-03-07T03:25:34.884328+0000 mgr.y (mgr.34107) 412 : cluster 0 pgmap v710: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 516 KiB/s rd, 8 op/s 2024-03-07T03:25:35.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:34 smithi145 bash[82329]: cluster 2024-03-07T03:25:34.884328+0000 mgr.y (mgr.34107) 412 : cluster 0 pgmap v710: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 516 KiB/s rd, 8 op/s 2024-03-07T03:25:35.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:34 smithi189 bash[66969]: cluster 2024-03-07T03:25:34.884328+0000 mgr.y (mgr.34107) 412 : cluster 0 pgmap v710: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 516 KiB/s rd, 8 op/s 2024-03-07T03:25:37.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:36 smithi145 bash[82329]: cluster 2024-03-07T03:25:36.886996+0000 mgr.y (mgr.34107) 413 : cluster 0 pgmap v711: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 498 KiB/s rd, 16 op/s 2024-03-07T03:25:37.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:36 smithi145 bash[79878]: cluster 2024-03-07T03:25:36.886996+0000 mgr.y (mgr.34107) 413 : cluster 0 pgmap v711: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 498 KiB/s rd, 16 op/s 2024-03-07T03:25:37.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:36 smithi189 bash[66969]: cluster 2024-03-07T03:25:36.886996+0000 mgr.y (mgr.34107) 413 : cluster 0 pgmap v711: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 498 KiB/s rd, 16 op/s 2024-03-07T03:25:39.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:38 smithi145 bash[79878]: cluster 2024-03-07T03:25:38.888179+0000 mgr.y (mgr.34107) 414 : cluster 0 pgmap v712: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:25:39.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:38 smithi145 bash[82329]: cluster 2024-03-07T03:25:38.888179+0000 mgr.y (mgr.34107) 414 : cluster 0 pgmap v712: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:25:39.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:38 smithi189 bash[66969]: cluster 2024-03-07T03:25:38.888179+0000 mgr.y (mgr.34107) 414 : cluster 0 pgmap v712: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:25:41.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:40 smithi145 bash[79878]: cluster 2024-03-07T03:25:40.889646+0000 mgr.y (mgr.34107) 415 : cluster 0 pgmap v713: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:25:41.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:40 smithi145 bash[82329]: cluster 2024-03-07T03:25:40.889646+0000 mgr.y (mgr.34107) 415 : cluster 0 pgmap v713: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:25:41.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:40 smithi189 bash[66969]: cluster 2024-03-07T03:25:40.889646+0000 mgr.y (mgr.34107) 415 : cluster 0 pgmap v713: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:25:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:25:42 smithi189 bash[59814]: ts=2024-03-07T03:25:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:25:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:42 smithi145 bash[79878]: cluster 2024-03-07T03:25:42.890681+0000 mgr.y (mgr.34107) 416 : cluster 0 pgmap v714: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 130 KiB/s rd, 85 B/s wr, 12 op/s 2024-03-07T03:25:43.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:25:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:25:42] "GET /metrics HTTP/1.1" 200 60646 "" "Prometheus/2.43.0" 2024-03-07T03:25:43.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:42 smithi145 bash[82329]: cluster 2024-03-07T03:25:42.890681+0000 mgr.y (mgr.34107) 416 : cluster 0 pgmap v714: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 130 KiB/s rd, 85 B/s wr, 12 op/s 2024-03-07T03:25:43.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:42 smithi189 bash[66969]: cluster 2024-03-07T03:25:42.890681+0000 mgr.y (mgr.34107) 416 : cluster 0 pgmap v714: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 130 KiB/s rd, 85 B/s wr, 12 op/s 2024-03-07T03:25:45.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:44 smithi145 bash[82329]: cluster 2024-03-07T03:25:44.892641+0000 mgr.y (mgr.34107) 417 : cluster 0 pgmap v715: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 85 B/s wr, 10 op/s 2024-03-07T03:25:45.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:44 smithi145 bash[79878]: cluster 2024-03-07T03:25:44.892641+0000 mgr.y (mgr.34107) 417 : cluster 0 pgmap v715: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 85 B/s wr, 10 op/s 2024-03-07T03:25:45.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:44 smithi189 bash[66969]: cluster 2024-03-07T03:25:44.892641+0000 mgr.y (mgr.34107) 417 : cluster 0 pgmap v715: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 85 B/s wr, 10 op/s 2024-03-07T03:25:47.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:46 smithi145 bash[79878]: cluster 2024-03-07T03:25:46.895414+0000 mgr.y (mgr.34107) 418 : cluster 0 pgmap v716: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:25:47.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:46 smithi145 bash[82329]: cluster 2024-03-07T03:25:46.895414+0000 mgr.y (mgr.34107) 418 : cluster 0 pgmap v716: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:25:47.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:46 smithi189 bash[66969]: cluster 2024-03-07T03:25:46.895414+0000 mgr.y (mgr.34107) 418 : cluster 0 pgmap v716: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:25:48.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:47 smithi145 bash[79878]: audit 2024-03-07T03:25:47.647183+0000 mon.a (mon.0) 1127 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:48.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:47 smithi145 bash[82329]: audit 2024-03-07T03:25:47.647183+0000 mon.a (mon.0) 1127 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:48.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:47 smithi189 bash[66969]: audit 2024-03-07T03:25:47.647183+0000 mon.a (mon.0) 1127 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:49.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:48 smithi145 bash[79878]: cluster 2024-03-07T03:25:48.896666+0000 mgr.y (mgr.34107) 419 : cluster 0 pgmap v717: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 175 KiB/s rd, 85 B/s wr, 9 op/s 2024-03-07T03:25:49.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:48 smithi145 bash[82329]: cluster 2024-03-07T03:25:48.896666+0000 mgr.y (mgr.34107) 419 : cluster 0 pgmap v717: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 175 KiB/s rd, 85 B/s wr, 9 op/s 2024-03-07T03:25:49.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:48 smithi189 bash[66969]: cluster 2024-03-07T03:25:48.896666+0000 mgr.y (mgr.34107) 419 : cluster 0 pgmap v717: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 175 KiB/s rd, 85 B/s wr, 9 op/s 2024-03-07T03:25:51.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:50 smithi145 bash[79878]: cluster 2024-03-07T03:25:50.897722+0000 mgr.y (mgr.34107) 420 : cluster 0 pgmap v718: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 170 B/s wr, 14 op/s 2024-03-07T03:25:51.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:50 smithi145 bash[82329]: cluster 2024-03-07T03:25:50.897722+0000 mgr.y (mgr.34107) 420 : cluster 0 pgmap v718: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 170 B/s wr, 14 op/s 2024-03-07T03:25:51.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:50 smithi189 bash[66969]: cluster 2024-03-07T03:25:50.897722+0000 mgr.y (mgr.34107) 420 : cluster 0 pgmap v718: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 170 B/s wr, 14 op/s 2024-03-07T03:25:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:25:52 smithi189 bash[59814]: ts=2024-03-07T03:25:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:25:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:52 smithi145 bash[79878]: cluster 2024-03-07T03:25:52.899428+0000 mgr.y (mgr.34107) 421 : cluster 0 pgmap v719: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:25:53.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:25:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:25:52] "GET /metrics HTTP/1.1" 200 60647 "" "Prometheus/2.43.0" 2024-03-07T03:25:53.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:52 smithi145 bash[82329]: cluster 2024-03-07T03:25:52.899428+0000 mgr.y (mgr.34107) 421 : cluster 0 pgmap v719: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:25:53.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:52 smithi189 bash[66969]: cluster 2024-03-07T03:25:52.899428+0000 mgr.y (mgr.34107) 421 : cluster 0 pgmap v719: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:25:55.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:54 smithi145 bash[82329]: cluster 2024-03-07T03:25:54.900568+0000 mgr.y (mgr.34107) 422 : cluster 0 pgmap v720: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 156 KiB/s rd, 170 B/s wr, 15 op/s 2024-03-07T03:25:55.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:54 smithi145 bash[79878]: cluster 2024-03-07T03:25:54.900568+0000 mgr.y (mgr.34107) 422 : cluster 0 pgmap v720: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 156 KiB/s rd, 170 B/s wr, 15 op/s 2024-03-07T03:25:55.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:54 smithi189 bash[66969]: cluster 2024-03-07T03:25:54.900568+0000 mgr.y (mgr.34107) 422 : cluster 0 pgmap v720: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 156 KiB/s rd, 170 B/s wr, 15 op/s 2024-03-07T03:25:57.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:56 smithi145 bash[82329]: cluster 2024-03-07T03:25:56.901997+0000 mgr.y (mgr.34107) 423 : cluster 0 pgmap v721: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 186 KiB/s rd, 170 B/s wr, 22 op/s 2024-03-07T03:25:57.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:56 smithi145 bash[79878]: cluster 2024-03-07T03:25:56.901997+0000 mgr.y (mgr.34107) 423 : cluster 0 pgmap v721: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 186 KiB/s rd, 170 B/s wr, 22 op/s 2024-03-07T03:25:57.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:56 smithi189 bash[66969]: cluster 2024-03-07T03:25:56.901997+0000 mgr.y (mgr.34107) 423 : cluster 0 pgmap v721: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 186 KiB/s rd, 170 B/s wr, 22 op/s 2024-03-07T03:25:59.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:25:58 smithi145 bash[82329]: cluster 2024-03-07T03:25:58.903000+0000 mgr.y (mgr.34107) 424 : cluster 0 pgmap v722: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:25:59.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:25:58 smithi145 bash[79878]: cluster 2024-03-07T03:25:58.903000+0000 mgr.y (mgr.34107) 424 : cluster 0 pgmap v722: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:25:59.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:25:58 smithi189 bash[66969]: cluster 2024-03-07T03:25:58.903000+0000 mgr.y (mgr.34107) 424 : cluster 0 pgmap v722: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:26:01.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:00 smithi145 bash[79878]: cluster 2024-03-07T03:26:00.904917+0000 mgr.y (mgr.34107) 425 : cluster 0 pgmap v723: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 170 B/s wr, 22 op/s 2024-03-07T03:26:01.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:00 smithi145 bash[82329]: cluster 2024-03-07T03:26:00.904917+0000 mgr.y (mgr.34107) 425 : cluster 0 pgmap v723: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 170 B/s wr, 22 op/s 2024-03-07T03:26:01.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:00 smithi189 bash[66969]: cluster 2024-03-07T03:26:00.904917+0000 mgr.y (mgr.34107) 425 : cluster 0 pgmap v723: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 170 B/s wr, 22 op/s 2024-03-07T03:26:02.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:02 smithi189 bash[66969]: audit 2024-03-07T03:26:02.510858+0000 mon.a (mon.0) 1128 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:26:02.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:26:02 smithi189 bash[59814]: ts=2024-03-07T03:26:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:26:02.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:02 smithi145 bash[82329]: audit 2024-03-07T03:26:02.510858+0000 mon.a (mon.0) 1128 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:26:02.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:02 smithi145 bash[79878]: audit 2024-03-07T03:26:02.510858+0000 mon.a (mon.0) 1128 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:26:02.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:26:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:26:02] "GET /metrics HTTP/1.1" 200 60643 "" "Prometheus/2.43.0" 2024-03-07T03:26:03.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:03 smithi189 bash[66969]: audit 2024-03-07T03:26:02.648101+0000 mon.a (mon.0) 1129 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:03.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:03 smithi189 bash[66969]: cluster 2024-03-07T03:26:02.906643+0000 mgr.y (mgr.34107) 426 : cluster 0 pgmap v724: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 85 B/s wr, 18 op/s 2024-03-07T03:26:04.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:03 smithi145 bash[82329]: audit 2024-03-07T03:26:02.648101+0000 mon.a (mon.0) 1129 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:04.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:03 smithi145 bash[82329]: cluster 2024-03-07T03:26:02.906643+0000 mgr.y (mgr.34107) 426 : cluster 0 pgmap v724: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 85 B/s wr, 18 op/s 2024-03-07T03:26:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:03 smithi145 bash[79878]: audit 2024-03-07T03:26:02.648101+0000 mon.a (mon.0) 1129 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:03 smithi145 bash[79878]: cluster 2024-03-07T03:26:02.906643+0000 mgr.y (mgr.34107) 426 : cluster 0 pgmap v724: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 85 B/s wr, 18 op/s 2024-03-07T03:26:05.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:04 smithi145 bash[79878]: cluster 2024-03-07T03:26:04.908617+0000 mgr.y (mgr.34107) 427 : cluster 0 pgmap v725: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:26:05.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:04 smithi145 bash[82329]: cluster 2024-03-07T03:26:04.908617+0000 mgr.y (mgr.34107) 427 : cluster 0 pgmap v725: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:26:05.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:04 smithi189 bash[66969]: cluster 2024-03-07T03:26:04.908617+0000 mgr.y (mgr.34107) 427 : cluster 0 pgmap v725: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:26:07.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:06 smithi145 bash[79878]: cluster 2024-03-07T03:26:06.910126+0000 mgr.y (mgr.34107) 428 : cluster 0 pgmap v726: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 23 op/s 2024-03-07T03:26:07.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:06 smithi145 bash[82329]: cluster 2024-03-07T03:26:06.910126+0000 mgr.y (mgr.34107) 428 : cluster 0 pgmap v726: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 23 op/s 2024-03-07T03:26:07.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:06 smithi189 bash[66969]: cluster 2024-03-07T03:26:06.910126+0000 mgr.y (mgr.34107) 428 : cluster 0 pgmap v726: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 23 op/s 2024-03-07T03:26:09.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[79878]: audit 2024-03-07T03:26:08.266343+0000 mon.a (mon.0) 1130 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[79878]: audit 2024-03-07T03:26:08.275758+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[79878]: audit 2024-03-07T03:26:08.576374+0000 mon.a (mon.0) 1132 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[79878]: audit 2024-03-07T03:26:08.586452+0000 mon.a (mon.0) 1133 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[79878]: cluster 2024-03-07T03:26:08.911241+0000 mgr.y (mgr.34107) 429 : cluster 0 pgmap v727: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 15 op/s 2024-03-07T03:26:09.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[79878]: audit 2024-03-07T03:26:09.220714+0000 mon.a (mon.0) 1134 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:26:09.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[79878]: audit 2024-03-07T03:26:09.222210+0000 mon.a (mon.0) 1135 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:26:09.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[79878]: audit 2024-03-07T03:26:09.231605+0000 mon.a (mon.0) 1136 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[82329]: audit 2024-03-07T03:26:08.266343+0000 mon.a (mon.0) 1130 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[82329]: audit 2024-03-07T03:26:08.275758+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[82329]: audit 2024-03-07T03:26:08.576374+0000 mon.a (mon.0) 1132 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[82329]: audit 2024-03-07T03:26:08.586452+0000 mon.a (mon.0) 1133 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[82329]: cluster 2024-03-07T03:26:08.911241+0000 mgr.y (mgr.34107) 429 : cluster 0 pgmap v727: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 15 op/s 2024-03-07T03:26:09.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[82329]: audit 2024-03-07T03:26:09.220714+0000 mon.a (mon.0) 1134 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:26:09.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[82329]: audit 2024-03-07T03:26:09.222210+0000 mon.a (mon.0) 1135 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:26:09.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:09 smithi145 bash[82329]: audit 2024-03-07T03:26:09.231605+0000 mon.a (mon.0) 1136 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:09 smithi189 bash[66969]: audit 2024-03-07T03:26:08.266343+0000 mon.a (mon.0) 1130 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:09 smithi189 bash[66969]: audit 2024-03-07T03:26:08.275758+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:09 smithi189 bash[66969]: audit 2024-03-07T03:26:08.576374+0000 mon.a (mon.0) 1132 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:09 smithi189 bash[66969]: audit 2024-03-07T03:26:08.586452+0000 mon.a (mon.0) 1133 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:09.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:09 smithi189 bash[66969]: cluster 2024-03-07T03:26:08.911241+0000 mgr.y (mgr.34107) 429 : cluster 0 pgmap v727: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 15 op/s 2024-03-07T03:26:09.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:09 smithi189 bash[66969]: audit 2024-03-07T03:26:09.220714+0000 mon.a (mon.0) 1134 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:26:09.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:09 smithi189 bash[66969]: audit 2024-03-07T03:26:09.222210+0000 mon.a (mon.0) 1135 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:26:09.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:09 smithi189 bash[66969]: audit 2024-03-07T03:26:09.231605+0000 mon.a (mon.0) 1136 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:26:11.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:10 smithi145 bash[79878]: cluster 2024-03-07T03:26:10.912251+0000 mgr.y (mgr.34107) 430 : cluster 0 pgmap v728: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 16 op/s 2024-03-07T03:26:11.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:10 smithi145 bash[82329]: cluster 2024-03-07T03:26:10.912251+0000 mgr.y (mgr.34107) 430 : cluster 0 pgmap v728: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 16 op/s 2024-03-07T03:26:11.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:10 smithi189 bash[66969]: cluster 2024-03-07T03:26:10.912251+0000 mgr.y (mgr.34107) 430 : cluster 0 pgmap v728: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 16 op/s 2024-03-07T03:26:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:26:12 smithi189 bash[59814]: ts=2024-03-07T03:26:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:26:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:12 smithi145 bash[79878]: cluster 2024-03-07T03:26:12.913298+0000 mgr.y (mgr.34107) 431 : cluster 0 pgmap v729: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 12 op/s 2024-03-07T03:26:13.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:26:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:26:12] "GET /metrics HTTP/1.1" 200 60643 "" "Prometheus/2.43.0" 2024-03-07T03:26:13.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:12 smithi145 bash[82329]: cluster 2024-03-07T03:26:12.913298+0000 mgr.y (mgr.34107) 431 : cluster 0 pgmap v729: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 12 op/s 2024-03-07T03:26:13.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:12 smithi189 bash[66969]: cluster 2024-03-07T03:26:12.913298+0000 mgr.y (mgr.34107) 431 : cluster 0 pgmap v729: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 12 op/s 2024-03-07T03:26:15.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:14 smithi145 bash[79878]: cluster 2024-03-07T03:26:14.914428+0000 mgr.y (mgr.34107) 432 : cluster 0 pgmap v730: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 10 op/s 2024-03-07T03:26:15.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:14 smithi145 bash[82329]: cluster 2024-03-07T03:26:14.914428+0000 mgr.y (mgr.34107) 432 : cluster 0 pgmap v730: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 10 op/s 2024-03-07T03:26:15.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:14 smithi189 bash[66969]: cluster 2024-03-07T03:26:14.914428+0000 mgr.y (mgr.34107) 432 : cluster 0 pgmap v730: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 10 op/s 2024-03-07T03:26:17.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:16 smithi145 bash[79878]: cluster 2024-03-07T03:26:16.916993+0000 mgr.y (mgr.34107) 433 : cluster 0 pgmap v731: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 206 KiB/s rd, 16 op/s 2024-03-07T03:26:17.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:16 smithi145 bash[82329]: cluster 2024-03-07T03:26:16.916993+0000 mgr.y (mgr.34107) 433 : cluster 0 pgmap v731: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 206 KiB/s rd, 16 op/s 2024-03-07T03:26:17.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:16 smithi189 bash[66969]: cluster 2024-03-07T03:26:16.916993+0000 mgr.y (mgr.34107) 433 : cluster 0 pgmap v731: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 206 KiB/s rd, 16 op/s 2024-03-07T03:26:18.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:17 smithi145 bash[79878]: audit 2024-03-07T03:26:17.647692+0000 mon.a (mon.0) 1137 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:18.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:17 smithi145 bash[82329]: audit 2024-03-07T03:26:17.647692+0000 mon.a (mon.0) 1137 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:18.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:17 smithi189 bash[66969]: audit 2024-03-07T03:26:17.647692+0000 mon.a (mon.0) 1137 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:19.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:18 smithi145 bash[79878]: cluster 2024-03-07T03:26:18.918366+0000 mgr.y (mgr.34107) 434 : cluster 0 pgmap v732: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 175 KiB/s rd, 9 op/s 2024-03-07T03:26:19.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:18 smithi145 bash[82329]: cluster 2024-03-07T03:26:18.918366+0000 mgr.y (mgr.34107) 434 : cluster 0 pgmap v732: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 175 KiB/s rd, 9 op/s 2024-03-07T03:26:19.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:18 smithi189 bash[66969]: cluster 2024-03-07T03:26:18.918366+0000 mgr.y (mgr.34107) 434 : cluster 0 pgmap v732: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 175 KiB/s rd, 9 op/s 2024-03-07T03:26:22.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:21 smithi189 bash[66969]: cluster 2024-03-07T03:26:20.920242+0000 mgr.y (mgr.34107) 435 : cluster 0 pgmap v733: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 15 op/s 2024-03-07T03:26:22.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:21 smithi145 bash[79878]: cluster 2024-03-07T03:26:20.920242+0000 mgr.y (mgr.34107) 435 : cluster 0 pgmap v733: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 15 op/s 2024-03-07T03:26:22.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:21 smithi145 bash[82329]: cluster 2024-03-07T03:26:20.920242+0000 mgr.y (mgr.34107) 435 : cluster 0 pgmap v733: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 15 op/s 2024-03-07T03:26:22.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:26:22 smithi189 bash[59814]: ts=2024-03-07T03:26:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:26:23.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:26:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:26:22] "GET /metrics HTTP/1.1" 200 60648 "" "Prometheus/2.43.0" 2024-03-07T03:26:24.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:23 smithi145 bash[79878]: cluster 2024-03-07T03:26:22.921994+0000 mgr.y (mgr.34107) 436 : cluster 0 pgmap v734: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:26:24.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:23 smithi145 bash[82329]: cluster 2024-03-07T03:26:22.921994+0000 mgr.y (mgr.34107) 436 : cluster 0 pgmap v734: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:26:24.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:23 smithi189 bash[66969]: cluster 2024-03-07T03:26:22.921994+0000 mgr.y (mgr.34107) 436 : cluster 0 pgmap v734: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:26:26.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:25 smithi145 bash[79878]: cluster 2024-03-07T03:26:24.923791+0000 mgr.y (mgr.34107) 437 : cluster 0 pgmap v735: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 156 KiB/s rd, 14 op/s 2024-03-07T03:26:26.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:25 smithi145 bash[82329]: cluster 2024-03-07T03:26:24.923791+0000 mgr.y (mgr.34107) 437 : cluster 0 pgmap v735: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 156 KiB/s rd, 14 op/s 2024-03-07T03:26:26.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:25 smithi189 bash[66969]: cluster 2024-03-07T03:26:24.923791+0000 mgr.y (mgr.34107) 437 : cluster 0 pgmap v735: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 156 KiB/s rd, 14 op/s 2024-03-07T03:26:28.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:27 smithi145 bash[79878]: cluster 2024-03-07T03:26:26.925160+0000 mgr.y (mgr.34107) 438 : cluster 0 pgmap v736: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 186 KiB/s rd, 22 op/s 2024-03-07T03:26:28.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:27 smithi145 bash[82329]: cluster 2024-03-07T03:26:26.925160+0000 mgr.y (mgr.34107) 438 : cluster 0 pgmap v736: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 186 KiB/s rd, 22 op/s 2024-03-07T03:26:28.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:27 smithi189 bash[66969]: cluster 2024-03-07T03:26:26.925160+0000 mgr.y (mgr.34107) 438 : cluster 0 pgmap v736: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 186 KiB/s rd, 22 op/s 2024-03-07T03:26:30.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:30 smithi145 bash[79878]: cluster 2024-03-07T03:26:28.926175+0000 mgr.y (mgr.34107) 439 : cluster 0 pgmap v737: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:26:30.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:30 smithi145 bash[82329]: cluster 2024-03-07T03:26:28.926175+0000 mgr.y (mgr.34107) 439 : cluster 0 pgmap v737: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:26:30.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:30 smithi189 bash[66969]: cluster 2024-03-07T03:26:28.926175+0000 mgr.y (mgr.34107) 439 : cluster 0 pgmap v737: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:26:32.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:32 smithi189 bash[66969]: cluster 2024-03-07T03:26:30.927324+0000 mgr.y (mgr.34107) 440 : cluster 0 pgmap v738: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 22 op/s 2024-03-07T03:26:32.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:32 smithi145 bash[79878]: cluster 2024-03-07T03:26:30.927324+0000 mgr.y (mgr.34107) 440 : cluster 0 pgmap v738: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 22 op/s 2024-03-07T03:26:32.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:32 smithi145 bash[82329]: cluster 2024-03-07T03:26:30.927324+0000 mgr.y (mgr.34107) 440 : cluster 0 pgmap v738: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 22 op/s 2024-03-07T03:26:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:26:32 smithi189 bash[59814]: ts=2024-03-07T03:26:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:26:33.002 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:26:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:26:32] "GET /metrics HTTP/1.1" 200 60644 "" "Prometheus/2.43.0" 2024-03-07T03:26:33.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:33 smithi145 bash[79878]: audit 2024-03-07T03:26:32.648687+0000 mon.a (mon.0) 1138 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:33.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:33 smithi145 bash[82329]: audit 2024-03-07T03:26:32.648687+0000 mon.a (mon.0) 1138 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:33.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:33 smithi189 bash[66969]: audit 2024-03-07T03:26:32.648687+0000 mon.a (mon.0) 1138 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:34.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:34 smithi145 bash[79878]: cluster 2024-03-07T03:26:32.928322+0000 mgr.y (mgr.34107) 441 : cluster 0 pgmap v739: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 18 op/s 2024-03-07T03:26:34.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:34 smithi145 bash[82329]: cluster 2024-03-07T03:26:32.928322+0000 mgr.y (mgr.34107) 441 : cluster 0 pgmap v739: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 18 op/s 2024-03-07T03:26:34.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:34 smithi189 bash[66969]: cluster 2024-03-07T03:26:32.928322+0000 mgr.y (mgr.34107) 441 : cluster 0 pgmap v739: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 18 op/s 2024-03-07T03:26:36.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:36 smithi145 bash[79878]: cluster 2024-03-07T03:26:34.929408+0000 mgr.y (mgr.34107) 442 : cluster 0 pgmap v740: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:26:36.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:36 smithi145 bash[82329]: cluster 2024-03-07T03:26:34.929408+0000 mgr.y (mgr.34107) 442 : cluster 0 pgmap v740: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:26:36.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:36 smithi189 bash[66969]: cluster 2024-03-07T03:26:34.929408+0000 mgr.y (mgr.34107) 442 : cluster 0 pgmap v740: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:26:38.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:38 smithi145 bash[79878]: cluster 2024-03-07T03:26:36.932187+0000 mgr.y (mgr.34107) 443 : cluster 0 pgmap v741: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 23 op/s 2024-03-07T03:26:38.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:38 smithi145 bash[82329]: cluster 2024-03-07T03:26:36.932187+0000 mgr.y (mgr.34107) 443 : cluster 0 pgmap v741: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 23 op/s 2024-03-07T03:26:38.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:38 smithi189 bash[66969]: cluster 2024-03-07T03:26:36.932187+0000 mgr.y (mgr.34107) 443 : cluster 0 pgmap v741: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 23 op/s 2024-03-07T03:26:40.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:40 smithi145 bash[79878]: cluster 2024-03-07T03:26:38.934268+0000 mgr.y (mgr.34107) 444 : cluster 0 pgmap v742: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 15 op/s 2024-03-07T03:26:40.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:40 smithi145 bash[82329]: cluster 2024-03-07T03:26:38.934268+0000 mgr.y (mgr.34107) 444 : cluster 0 pgmap v742: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 15 op/s 2024-03-07T03:26:40.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:40 smithi189 bash[66969]: cluster 2024-03-07T03:26:38.934268+0000 mgr.y (mgr.34107) 444 : cluster 0 pgmap v742: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 15 op/s 2024-03-07T03:26:42.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:42 smithi189 bash[66969]: cluster 2024-03-07T03:26:40.935399+0000 mgr.y (mgr.34107) 445 : cluster 0 pgmap v743: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:26:42.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:42 smithi145 bash[79878]: cluster 2024-03-07T03:26:40.935399+0000 mgr.y (mgr.34107) 445 : cluster 0 pgmap v743: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:26:42.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:42 smithi145 bash[82329]: cluster 2024-03-07T03:26:40.935399+0000 mgr.y (mgr.34107) 445 : cluster 0 pgmap v743: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:26:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:26:42 smithi189 bash[59814]: ts=2024-03-07T03:26:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:26:43.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:26:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:26:42] "GET /metrics HTTP/1.1" 200 60644 "" "Prometheus/2.43.0" 2024-03-07T03:26:44.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:44 smithi145 bash[82329]: cluster 2024-03-07T03:26:42.937110+0000 mgr.y (mgr.34107) 446 : cluster 0 pgmap v744: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 85 B/s wr, 12 op/s 2024-03-07T03:26:44.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:44 smithi145 bash[79878]: cluster 2024-03-07T03:26:42.937110+0000 mgr.y (mgr.34107) 446 : cluster 0 pgmap v744: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 85 B/s wr, 12 op/s 2024-03-07T03:26:44.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:44 smithi189 bash[66969]: cluster 2024-03-07T03:26:42.937110+0000 mgr.y (mgr.34107) 446 : cluster 0 pgmap v744: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 85 B/s wr, 12 op/s 2024-03-07T03:26:46.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:46 smithi145 bash[79878]: cluster 2024-03-07T03:26:44.939104+0000 mgr.y (mgr.34107) 447 : cluster 0 pgmap v745: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 85 B/s wr, 10 op/s 2024-03-07T03:26:46.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:46 smithi145 bash[82329]: cluster 2024-03-07T03:26:44.939104+0000 mgr.y (mgr.34107) 447 : cluster 0 pgmap v745: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 85 B/s wr, 10 op/s 2024-03-07T03:26:46.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:46 smithi189 bash[66969]: cluster 2024-03-07T03:26:44.939104+0000 mgr.y (mgr.34107) 447 : cluster 0 pgmap v745: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 85 B/s wr, 10 op/s 2024-03-07T03:26:47.124 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.EncryptionLoadFormatMismatch (129410 ms) 2024-03-07T03:26:47.124 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.EncryptedResize 2024-03-07T03:26:47.124 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:26:48.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:48 smithi145 bash[79878]: cluster 2024-03-07T03:26:46.941712+0000 mgr.y (mgr.34107) 448 : cluster 0 pgmap v746: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 205 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:26:48.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:48 smithi145 bash[79878]: audit 2024-03-07T03:26:47.649057+0000 mon.a (mon.0) 1139 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:48.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:48 smithi145 bash[82329]: cluster 2024-03-07T03:26:46.941712+0000 mgr.y (mgr.34107) 448 : cluster 0 pgmap v746: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 205 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:26:48.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:48 smithi145 bash[82329]: audit 2024-03-07T03:26:47.649057+0000 mon.a (mon.0) 1139 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:48.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:48 smithi189 bash[66969]: cluster 2024-03-07T03:26:46.941712+0000 mgr.y (mgr.34107) 448 : cluster 0 pgmap v746: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 205 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:26:48.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:48 smithi189 bash[66969]: audit 2024-03-07T03:26:47.649057+0000 mon.a (mon.0) 1139 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:50.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:50 smithi145 bash[79878]: cluster 2024-03-07T03:26:48.943191+0000 mgr.y (mgr.34107) 449 : cluster 0 pgmap v747: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 340 B/s wr, 12 op/s 2024-03-07T03:26:50.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:50 smithi145 bash[82329]: cluster 2024-03-07T03:26:48.943191+0000 mgr.y (mgr.34107) 449 : cluster 0 pgmap v747: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 340 B/s wr, 12 op/s 2024-03-07T03:26:50.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:50 smithi189 bash[66969]: cluster 2024-03-07T03:26:48.943191+0000 mgr.y (mgr.34107) 449 : cluster 0 pgmap v747: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 340 B/s wr, 12 op/s 2024-03-07T03:26:52.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:52 smithi189 bash[66969]: cluster 2024-03-07T03:26:50.944701+0000 mgr.y (mgr.34107) 450 : cluster 0 pgmap v748: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 341 B/s wr, 12 op/s 2024-03-07T03:26:52.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:52 smithi145 bash[79878]: cluster 2024-03-07T03:26:50.944701+0000 mgr.y (mgr.34107) 450 : cluster 0 pgmap v748: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 341 B/s wr, 12 op/s 2024-03-07T03:26:52.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:52 smithi145 bash[82329]: cluster 2024-03-07T03:26:50.944701+0000 mgr.y (mgr.34107) 450 : cluster 0 pgmap v748: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 341 B/s wr, 12 op/s 2024-03-07T03:26:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:26:52 smithi189 bash[59814]: ts=2024-03-07T03:26:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:26:53.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:26:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:26:52] "GET /metrics HTTP/1.1" 200 60646 "" "Prometheus/2.43.0" 2024-03-07T03:26:54.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:54 smithi145 bash[79878]: cluster 2024-03-07T03:26:52.945910+0000 mgr.y (mgr.34107) 451 : cluster 0 pgmap v749: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 341 B/s wr, 11 op/s 2024-03-07T03:26:54.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:54 smithi145 bash[82329]: cluster 2024-03-07T03:26:52.945910+0000 mgr.y (mgr.34107) 451 : cluster 0 pgmap v749: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 341 B/s wr, 11 op/s 2024-03-07T03:26:54.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:54 smithi189 bash[66969]: cluster 2024-03-07T03:26:52.945910+0000 mgr.y (mgr.34107) 451 : cluster 0 pgmap v749: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 341 B/s wr, 11 op/s 2024-03-07T03:26:56.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:56 smithi145 bash[79878]: cluster 2024-03-07T03:26:54.947252+0000 mgr.y (mgr.34107) 452 : cluster 0 pgmap v750: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 341 B/s wr, 9 op/s 2024-03-07T03:26:56.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:56 smithi145 bash[82329]: cluster 2024-03-07T03:26:54.947252+0000 mgr.y (mgr.34107) 452 : cluster 0 pgmap v750: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 341 B/s wr, 9 op/s 2024-03-07T03:26:56.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:56 smithi189 bash[66969]: cluster 2024-03-07T03:26:54.947252+0000 mgr.y (mgr.34107) 452 : cluster 0 pgmap v750: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 341 B/s wr, 9 op/s 2024-03-07T03:26:58.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:26:58 smithi145 bash[79878]: cluster 2024-03-07T03:26:56.949236+0000 mgr.y (mgr.34107) 453 : cluster 0 pgmap v751: 425 pgs: 425 active+clean; 153 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 342 KiB/s wr, 12 op/s 2024-03-07T03:26:58.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:26:58 smithi145 bash[82329]: cluster 2024-03-07T03:26:56.949236+0000 mgr.y (mgr.34107) 453 : cluster 0 pgmap v751: 425 pgs: 425 active+clean; 153 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 342 KiB/s wr, 12 op/s 2024-03-07T03:26:58.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:26:58 smithi189 bash[66969]: cluster 2024-03-07T03:26:56.949236+0000 mgr.y (mgr.34107) 453 : cluster 0 pgmap v751: 425 pgs: 425 active+clean; 153 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 342 KiB/s wr, 12 op/s 2024-03-07T03:27:00.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:00 smithi145 bash[79878]: cluster 2024-03-07T03:26:58.950598+0000 mgr.y (mgr.34107) 454 : cluster 0 pgmap v752: 425 pgs: 425 active+clean; 165 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:27:00.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:00 smithi145 bash[82329]: cluster 2024-03-07T03:26:58.950598+0000 mgr.y (mgr.34107) 454 : cluster 0 pgmap v752: 425 pgs: 425 active+clean; 165 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:27:00.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:00 smithi189 bash[66969]: cluster 2024-03-07T03:26:58.950598+0000 mgr.y (mgr.34107) 454 : cluster 0 pgmap v752: 425 pgs: 425 active+clean; 165 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:27:01.723 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.EncryptedResize (14599 ms) 2024-03-07T03:27:01.723 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.EncryptedFlattenSmallData 2024-03-07T03:27:01.723 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:02.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:02 smithi145 bash[79878]: cluster 2024-03-07T03:27:00.951970+0000 mgr.y (mgr.34107) 455 : cluster 0 pgmap v753: 425 pgs: 425 active+clean; 169 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-03-07T03:27:02.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:02 smithi145 bash[82329]: cluster 2024-03-07T03:27:00.951970+0000 mgr.y (mgr.34107) 455 : cluster 0 pgmap v753: 425 pgs: 425 active+clean; 169 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-03-07T03:27:02.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:02 smithi189 bash[66969]: cluster 2024-03-07T03:27:00.951970+0000 mgr.y (mgr.34107) 455 : cluster 0 pgmap v753: 425 pgs: 425 active+clean; 169 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-03-07T03:27:02.329 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:27:02 smithi189 bash[59814]: ts=2024-03-07T03:27:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:27:03.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:27:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:27:02] "GET /metrics HTTP/1.1" 200 60656 "" "Prometheus/2.43.0" 2024-03-07T03:27:03.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:03 smithi189 bash[66969]: audit 2024-03-07T03:27:02.648500+0000 mon.a (mon.0) 1140 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:03.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:03 smithi145 bash[79878]: audit 2024-03-07T03:27:02.648500+0000 mon.a (mon.0) 1140 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:03.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:03 smithi145 bash[82329]: audit 2024-03-07T03:27:02.648500+0000 mon.a (mon.0) 1140 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:04.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:04 smithi189 bash[66969]: cluster 2024-03-07T03:27:02.953294+0000 mgr.y (mgr.34107) 456 : cluster 0 pgmap v754: 425 pgs: 425 active+clean; 169 MiB data, 867 MiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 1.6 MiB/s wr, 16 op/s 2024-03-07T03:27:04.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:04 smithi145 bash[79878]: cluster 2024-03-07T03:27:02.953294+0000 mgr.y (mgr.34107) 456 : cluster 0 pgmap v754: 425 pgs: 425 active+clean; 169 MiB data, 867 MiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 1.6 MiB/s wr, 16 op/s 2024-03-07T03:27:04.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:04 smithi145 bash[82329]: cluster 2024-03-07T03:27:02.953294+0000 mgr.y (mgr.34107) 456 : cluster 0 pgmap v754: 425 pgs: 425 active+clean; 169 MiB data, 867 MiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 1.6 MiB/s wr, 16 op/s 2024-03-07T03:27:06.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:06 smithi189 bash[66969]: cluster 2024-03-07T03:27:04.954680+0000 mgr.y (mgr.34107) 457 : cluster 0 pgmap v755: 425 pgs: 425 active+clean; 165 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 90 KiB/s rd, 1.6 MiB/s wr, 26 op/s 2024-03-07T03:27:06.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:06 smithi145 bash[79878]: cluster 2024-03-07T03:27:04.954680+0000 mgr.y (mgr.34107) 457 : cluster 0 pgmap v755: 425 pgs: 425 active+clean; 165 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 90 KiB/s rd, 1.6 MiB/s wr, 26 op/s 2024-03-07T03:27:06.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:06 smithi145 bash[82329]: cluster 2024-03-07T03:27:04.954680+0000 mgr.y (mgr.34107) 457 : cluster 0 pgmap v755: 425 pgs: 425 active+clean; 165 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 90 KiB/s rd, 1.6 MiB/s wr, 26 op/s 2024-03-07T03:27:08.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:08 smithi189 bash[66969]: cluster 2024-03-07T03:27:06.956439+0000 mgr.y (mgr.34107) 458 : cluster 0 pgmap v756: 425 pgs: 425 active+clean; 165 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 1.6 MiB/s wr, 29 op/s 2024-03-07T03:27:08.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:08 smithi145 bash[79878]: cluster 2024-03-07T03:27:06.956439+0000 mgr.y (mgr.34107) 458 : cluster 0 pgmap v756: 425 pgs: 425 active+clean; 165 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 1.6 MiB/s wr, 29 op/s 2024-03-07T03:27:08.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:08 smithi145 bash[82329]: cluster 2024-03-07T03:27:06.956439+0000 mgr.y (mgr.34107) 458 : cluster 0 pgmap v756: 425 pgs: 425 active+clean; 165 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 1.6 MiB/s wr, 29 op/s 2024-03-07T03:27:10.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:10 smithi145 bash[79878]: cluster 2024-03-07T03:27:08.957798+0000 mgr.y (mgr.34107) 459 : cluster 0 pgmap v757: 425 pgs: 425 active+clean; 165 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 1.3 MiB/s wr, 25 op/s 2024-03-07T03:27:10.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:10 smithi145 bash[79878]: audit 2024-03-07T03:27:09.319293+0000 mon.a (mon.0) 1141 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:27:10.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:10 smithi145 bash[82329]: cluster 2024-03-07T03:27:08.957798+0000 mgr.y (mgr.34107) 459 : cluster 0 pgmap v757: 425 pgs: 425 active+clean; 165 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 1.3 MiB/s wr, 25 op/s 2024-03-07T03:27:10.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:10 smithi145 bash[82329]: audit 2024-03-07T03:27:09.319293+0000 mon.a (mon.0) 1141 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:27:10.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:10 smithi189 bash[66969]: cluster 2024-03-07T03:27:08.957798+0000 mgr.y (mgr.34107) 459 : cluster 0 pgmap v757: 425 pgs: 425 active+clean; 165 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 1.3 MiB/s wr, 25 op/s 2024-03-07T03:27:10.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:10 smithi189 bash[66969]: audit 2024-03-07T03:27:09.319293+0000 mon.a (mon.0) 1141 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:27:11.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:11 smithi145 bash[79878]: cluster 2024-03-07T03:27:10.099409+0000 mon.a (mon.0) 1142 : cluster 0 osdmap e674: 8 total, 8 up, 8 in 2024-03-07T03:27:11.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:11 smithi145 bash[82329]: cluster 2024-03-07T03:27:10.099409+0000 mon.a (mon.0) 1142 : cluster 0 osdmap e674: 8 total, 8 up, 8 in 2024-03-07T03:27:11.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:11 smithi189 bash[66969]: cluster 2024-03-07T03:27:10.099409+0000 mon.a (mon.0) 1142 : cluster 0 osdmap e674: 8 total, 8 up, 8 in 2024-03-07T03:27:12.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:12 smithi145 bash[79878]: cluster 2024-03-07T03:27:10.958950+0000 mgr.y (mgr.34107) 460 : cluster 0 pgmap v759: 425 pgs: 425 active+clean; 173 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 820 KiB/s wr, 21 op/s 2024-03-07T03:27:12.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:12 smithi145 bash[82329]: cluster 2024-03-07T03:27:10.958950+0000 mgr.y (mgr.34107) 460 : cluster 0 pgmap v759: 425 pgs: 425 active+clean; 173 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 820 KiB/s wr, 21 op/s 2024-03-07T03:27:12.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:12 smithi189 bash[66969]: cluster 2024-03-07T03:27:10.958950+0000 mgr.y (mgr.34107) 460 : cluster 0 pgmap v759: 425 pgs: 425 active+clean; 173 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 820 KiB/s wr, 21 op/s 2024-03-07T03:27:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:27:12 smithi189 bash[59814]: ts=2024-03-07T03:27:12.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:27:13.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:27:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:27:12] "GET /metrics HTTP/1.1" 200 60656 "" "Prometheus/2.43.0" 2024-03-07T03:27:14.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:14 smithi145 bash[79878]: cluster 2024-03-07T03:27:12.960135+0000 mgr.y (mgr.34107) 461 : cluster 0 pgmap v760: 425 pgs: 425 active+clean; 181 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 1.6 MiB/s wr, 33 op/s 2024-03-07T03:27:14.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:14 smithi145 bash[82329]: cluster 2024-03-07T03:27:12.960135+0000 mgr.y (mgr.34107) 461 : cluster 0 pgmap v760: 425 pgs: 425 active+clean; 181 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 1.6 MiB/s wr, 33 op/s 2024-03-07T03:27:14.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:14 smithi189 bash[66969]: cluster 2024-03-07T03:27:12.960135+0000 mgr.y (mgr.34107) 461 : cluster 0 pgmap v760: 425 pgs: 425 active+clean; 181 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 1.6 MiB/s wr, 33 op/s 2024-03-07T03:27:16.286 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.EncryptedFlattenSmallData (14563 ms) 2024-03-07T03:27:16.286 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestIOWithIOHint 2024-03-07T03:27:16.286 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:16.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[79878]: cluster 2024-03-07T03:27:14.961322+0000 mgr.y (mgr.34107) 462 : cluster 0 pgmap v761: 425 pgs: 425 active+clean; 181 MiB data, 909 MiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 1.6 MiB/s wr, 33 op/s 2024-03-07T03:27:16.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[79878]: audit 2024-03-07T03:27:15.204495+0000 mon.a (mon.0) 1143 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[79878]: audit 2024-03-07T03:27:15.212145+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[79878]: audit 2024-03-07T03:27:15.358224+0000 mon.a (mon.0) 1145 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[79878]: audit 2024-03-07T03:27:15.367263+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[79878]: audit 2024-03-07T03:27:16.003287+0000 mon.a (mon.0) 1147 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:27:16.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[79878]: audit 2024-03-07T03:27:16.004161+0000 mon.a (mon.0) 1148 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:27:16.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[79878]: audit 2024-03-07T03:27:16.010443+0000 mon.a (mon.0) 1149 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[82329]: cluster 2024-03-07T03:27:14.961322+0000 mgr.y (mgr.34107) 462 : cluster 0 pgmap v761: 425 pgs: 425 active+clean; 181 MiB data, 909 MiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 1.6 MiB/s wr, 33 op/s 2024-03-07T03:27:16.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[82329]: audit 2024-03-07T03:27:15.204495+0000 mon.a (mon.0) 1143 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[82329]: audit 2024-03-07T03:27:15.212145+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[82329]: audit 2024-03-07T03:27:15.358224+0000 mon.a (mon.0) 1145 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[82329]: audit 2024-03-07T03:27:15.367263+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[82329]: audit 2024-03-07T03:27:16.003287+0000 mon.a (mon.0) 1147 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:27:16.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[82329]: audit 2024-03-07T03:27:16.004161+0000 mon.a (mon.0) 1148 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:27:16.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:16 smithi145 bash[82329]: audit 2024-03-07T03:27:16.010443+0000 mon.a (mon.0) 1149 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:16 smithi189 bash[66969]: cluster 2024-03-07T03:27:14.961322+0000 mgr.y (mgr.34107) 462 : cluster 0 pgmap v761: 425 pgs: 425 active+clean; 181 MiB data, 909 MiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 1.6 MiB/s wr, 33 op/s 2024-03-07T03:27:16.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:16 smithi189 bash[66969]: audit 2024-03-07T03:27:15.204495+0000 mon.a (mon.0) 1143 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:16 smithi189 bash[66969]: audit 2024-03-07T03:27:15.212145+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:16 smithi189 bash[66969]: audit 2024-03-07T03:27:15.358224+0000 mon.a (mon.0) 1145 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:16 smithi189 bash[66969]: audit 2024-03-07T03:27:15.367263+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:16.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:16 smithi189 bash[66969]: audit 2024-03-07T03:27:16.003287+0000 mon.a (mon.0) 1147 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:27:16.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:16 smithi189 bash[66969]: audit 2024-03-07T03:27:16.004161+0000 mon.a (mon.0) 1148 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:27:16.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:16 smithi189 bash[66969]: audit 2024-03-07T03:27:16.010443+0000 mon.a (mon.0) 1149 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:18.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:18 smithi145 bash[82329]: cluster 2024-03-07T03:27:16.964253+0000 mgr.y (mgr.34107) 463 : cluster 0 pgmap v762: 425 pgs: 425 active+clean; 193 MiB data, 941 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-03-07T03:27:18.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:18 smithi145 bash[82329]: audit 2024-03-07T03:27:17.650199+0000 mon.a (mon.0) 1150 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:18.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:18 smithi145 bash[79878]: cluster 2024-03-07T03:27:16.964253+0000 mgr.y (mgr.34107) 463 : cluster 0 pgmap v762: 425 pgs: 425 active+clean; 193 MiB data, 941 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-03-07T03:27:18.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:18 smithi145 bash[79878]: audit 2024-03-07T03:27:17.650199+0000 mon.a (mon.0) 1150 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:18.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:18 smithi189 bash[66969]: cluster 2024-03-07T03:27:16.964253+0000 mgr.y (mgr.34107) 463 : cluster 0 pgmap v762: 425 pgs: 425 active+clean; 193 MiB data, 941 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-03-07T03:27:18.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:18 smithi189 bash[66969]: audit 2024-03-07T03:27:17.650199+0000 mon.a (mon.0) 1150 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:20.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:20 smithi145 bash[79878]: cluster 2024-03-07T03:27:18.966542+0000 mgr.y (mgr.34107) 464 : cluster 0 pgmap v763: 425 pgs: 425 active+clean; 197 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 227 op/s 2024-03-07T03:27:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:20 smithi145 bash[82329]: cluster 2024-03-07T03:27:18.966542+0000 mgr.y (mgr.34107) 464 : cluster 0 pgmap v763: 425 pgs: 425 active+clean; 197 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 227 op/s 2024-03-07T03:27:20.579 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:27:20 smithi189 bash[49584]: logger=cleanup t=2024-03-07T03:27:20.34173583Z level=info msg="Completed cleanup jobs" duration=192.121438ms 2024-03-07T03:27:20.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:20 smithi189 bash[66969]: cluster 2024-03-07T03:27:18.966542+0000 mgr.y (mgr.34107) 464 : cluster 0 pgmap v763: 425 pgs: 425 active+clean; 197 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 227 op/s 2024-03-07T03:27:22.291 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:22.291 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:22.292 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:22.292 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:22.292 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:22.292 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.292 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:22.292 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.292 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.292 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:22.292 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.292 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:22.292 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.292 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.292 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.293 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:27:22.294 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.295 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:22.296 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:22.297 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 512~512 = 512 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:aio discard: 1536~512 = 0 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:22.298 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 0 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 0 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 16384 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 16384 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:22.299 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 49152 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 49152 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 65536 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 65536 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 98304 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.300 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 98304 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 114688 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 114688 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 147456 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.301 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 147456 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 163840 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 163840 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 196608 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 196608 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.302 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 212992 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 212992 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:22.303 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.304 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:22.305 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.306 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.306 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:22.306 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.306 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.306 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.306 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:22.306 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:22.306 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:22.306 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:22.306 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:22.306 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:22.306 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:22.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:22 smithi145 bash[79878]: cluster 2024-03-07T03:27:20.969076+0000 mgr.y (mgr.34107) 465 : cluster 0 pgmap v764: 425 pgs: 425 active+clean; 197 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.6 MiB/s wr, 212 op/s 2024-03-07T03:27:22.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:22 smithi145 bash[82329]: cluster 2024-03-07T03:27:20.969076+0000 mgr.y (mgr.34107) 465 : cluster 0 pgmap v764: 425 pgs: 425 active+clean; 197 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.6 MiB/s wr, 212 op/s 2024-03-07T03:27:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:22 smithi189 bash[66969]: cluster 2024-03-07T03:27:20.969076+0000 mgr.y (mgr.34107) 465 : cluster 0 pgmap v764: 425 pgs: 425 active+clean; 197 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.6 MiB/s wr, 212 op/s 2024-03-07T03:27:22.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:27:22 smithi189 bash[59814]: ts=2024-03-07T03:27:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:27:23.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:27:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:27:22] "GET /metrics HTTP/1.1" 200 60653 "" "Prometheus/2.43.0" 2024-03-07T03:27:24.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:24 smithi145 bash[79878]: cluster 2024-03-07T03:27:22.970971+0000 mgr.y (mgr.34107) 466 : cluster 0 pgmap v765: 425 pgs: 425 active+clean; 197 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.0 MiB/s wr, 190 op/s 2024-03-07T03:27:24.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:24 smithi145 bash[82329]: cluster 2024-03-07T03:27:22.970971+0000 mgr.y (mgr.34107) 466 : cluster 0 pgmap v765: 425 pgs: 425 active+clean; 197 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.0 MiB/s wr, 190 op/s 2024-03-07T03:27:24.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:24 smithi189 bash[66969]: cluster 2024-03-07T03:27:22.970971+0000 mgr.y (mgr.34107) 466 : cluster 0 pgmap v765: 425 pgs: 425 active+clean; 197 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.0 MiB/s wr, 190 op/s 2024-03-07T03:27:26.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:26 smithi145 bash[79878]: cluster 2024-03-07T03:27:24.973376+0000 mgr.y (mgr.34107) 467 : cluster 0 pgmap v766: 425 pgs: 425 active+clean; 197 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.4 MiB/s wr, 487 op/s 2024-03-07T03:27:26.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:26 smithi145 bash[82329]: cluster 2024-03-07T03:27:24.973376+0000 mgr.y (mgr.34107) 467 : cluster 0 pgmap v766: 425 pgs: 425 active+clean; 197 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.4 MiB/s wr, 487 op/s 2024-03-07T03:27:26.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:26 smithi189 bash[66969]: cluster 2024-03-07T03:27:24.973376+0000 mgr.y (mgr.34107) 467 : cluster 0 pgmap v766: 425 pgs: 425 active+clean; 197 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.4 MiB/s wr, 487 op/s 2024-03-07T03:27:27.519 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.520 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.520 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.520 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.520 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.520 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.520 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.520 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.520 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.520 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.520 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.520 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.520 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.521 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.521 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.521 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:27.521 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.521 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.521 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:27.521 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.521 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.521 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.521 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.521 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.521 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.522 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.522 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.522 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.522 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.522 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.522 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.522 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.522 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.522 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.522 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.522 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.522 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.522 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:27.523 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.524 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.525 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.525 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.525 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.525 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.525 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.525 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.525 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.525 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.525 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.525 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.525 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.525 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.525 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.526 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:27.527 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:27.528 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:27.528 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:27.528 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.528 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.528 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:27.528 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:27.528 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:27.528 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:27.528 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:27.528 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:27.528 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:27.528 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestIOWithIOHint (11233 ms) 2024-03-07T03:27:27.528 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestDataPoolIO 2024-03-07T03:27:28.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:28 smithi145 bash[79878]: cluster 2024-03-07T03:27:26.976127+0000 mgr.y (mgr.34107) 468 : cluster 0 pgmap v767: 425 pgs: 425 active+clean; 197 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.4 MiB/s wr, 478 op/s 2024-03-07T03:27:28.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:28 smithi145 bash[82329]: cluster 2024-03-07T03:27:26.976127+0000 mgr.y (mgr.34107) 468 : cluster 0 pgmap v767: 425 pgs: 425 active+clean; 197 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.4 MiB/s wr, 478 op/s 2024-03-07T03:27:28.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:28 smithi189 bash[66969]: cluster 2024-03-07T03:27:26.976127+0000 mgr.y (mgr.34107) 468 : cluster 0 pgmap v767: 425 pgs: 425 active+clean; 197 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.4 MiB/s wr, 478 op/s 2024-03-07T03:27:29.265 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:29.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:29 smithi145 bash[79878]: cluster 2024-03-07T03:27:28.244905+0000 mon.a (mon.0) 1151 : cluster 0 osdmap e675: 8 total, 8 up, 8 in 2024-03-07T03:27:29.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:29 smithi145 bash[79878]: audit 2024-03-07T03:27:28.248075+0000 mon.a (mon.0) 1152 : audit 1 from='client.? 172.21.15.145:0/1122940787' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:27:29.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:29 smithi145 bash[82329]: cluster 2024-03-07T03:27:28.244905+0000 mon.a (mon.0) 1151 : cluster 0 osdmap e675: 8 total, 8 up, 8 in 2024-03-07T03:27:29.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:29 smithi145 bash[82329]: audit 2024-03-07T03:27:28.248075+0000 mon.a (mon.0) 1152 : audit 1 from='client.? 172.21.15.145:0/1122940787' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:27:29.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:29 smithi189 bash[66969]: cluster 2024-03-07T03:27:28.244905+0000 mon.a (mon.0) 1151 : cluster 0 osdmap e675: 8 total, 8 up, 8 in 2024-03-07T03:27:29.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:29 smithi189 bash[66969]: audit 2024-03-07T03:27:28.248075+0000 mon.a (mon.0) 1152 : audit 1 from='client.? 172.21.15.145:0/1122940787' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:27:30.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:30 smithi145 bash[82329]: cluster 2024-03-07T03:27:28.978312+0000 mgr.y (mgr.34107) 469 : cluster 0 pgmap v769: 457 pgs: 10 creating+peering, 22 unknown, 425 active+clean; 199 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 323 KiB/s rd, 160 KiB/s wr, 659 op/s 2024-03-07T03:27:30.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:30 smithi145 bash[82329]: cluster 2024-03-07T03:27:29.239267+0000 mon.a (mon.0) 1153 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:27:30.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:30 smithi145 bash[82329]: audit 2024-03-07T03:27:29.252314+0000 mon.a (mon.0) 1154 : audit 1 from='client.? 172.21.15.145:0/1122940787' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:27:30.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:30 smithi145 bash[82329]: cluster 2024-03-07T03:27:29.263346+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e676: 8 total, 8 up, 8 in 2024-03-07T03:27:30.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:30 smithi145 bash[79878]: cluster 2024-03-07T03:27:28.978312+0000 mgr.y (mgr.34107) 469 : cluster 0 pgmap v769: 457 pgs: 10 creating+peering, 22 unknown, 425 active+clean; 199 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 323 KiB/s rd, 160 KiB/s wr, 659 op/s 2024-03-07T03:27:30.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:30 smithi145 bash[79878]: cluster 2024-03-07T03:27:29.239267+0000 mon.a (mon.0) 1153 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:27:30.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:30 smithi145 bash[79878]: audit 2024-03-07T03:27:29.252314+0000 mon.a (mon.0) 1154 : audit 1 from='client.? 172.21.15.145:0/1122940787' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:27:30.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:30 smithi145 bash[79878]: cluster 2024-03-07T03:27:29.263346+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e676: 8 total, 8 up, 8 in 2024-03-07T03:27:30.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:30 smithi189 bash[66969]: cluster 2024-03-07T03:27:28.978312+0000 mgr.y (mgr.34107) 469 : cluster 0 pgmap v769: 457 pgs: 10 creating+peering, 22 unknown, 425 active+clean; 199 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 323 KiB/s rd, 160 KiB/s wr, 659 op/s 2024-03-07T03:27:30.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:30 smithi189 bash[66969]: cluster 2024-03-07T03:27:29.239267+0000 mon.a (mon.0) 1153 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:27:30.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:30 smithi189 bash[66969]: audit 2024-03-07T03:27:29.252314+0000 mon.a (mon.0) 1154 : audit 1 from='client.? 172.21.15.145:0/1122940787' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:27:30.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:30 smithi189 bash[66969]: cluster 2024-03-07T03:27:29.263346+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e676: 8 total, 8 up, 8 in 2024-03-07T03:27:31.435 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:31.435 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:31.435 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:31.436 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:31.436 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:31.436 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:31.436 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:31.436 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:31.436 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:31.436 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:31.436 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:31.436 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:31.436 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:31.436 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:31.436 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:31.436 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:31.437 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:31.438 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 512~512 = 512 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:aio discard: 1536~512 = 0 2024-03-07T03:27:31.439 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:31.440 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:31.440 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:31.440 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:31.440 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:31.440 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:31.440 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:31.440 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestDataPoolIO (3916 ms) 2024-03-07T03:27:31.440 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatch 2024-03-07T03:27:31.440 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:31.504 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatch (69 ms) 2024-03-07T03:27:31.504 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatch 2024-03-07T03:27:31.504 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:31.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:31 smithi145 bash[79878]: cluster 2024-03-07T03:27:30.262742+0000 mon.a (mon.0) 1156 : cluster 0 osdmap e677: 8 total, 8 up, 8 in 2024-03-07T03:27:31.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:31 smithi145 bash[82329]: cluster 2024-03-07T03:27:30.262742+0000 mon.a (mon.0) 1156 : cluster 0 osdmap e677: 8 total, 8 up, 8 in 2024-03-07T03:27:31.577 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatch (73 ms) 2024-03-07T03:27:31.577 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccess 2024-03-07T03:27:31.577 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:31.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:31 smithi189 bash[66969]: cluster 2024-03-07T03:27:30.262742+0000 mon.a (mon.0) 1156 : cluster 0 osdmap e677: 8 total, 8 up, 8 in 2024-03-07T03:27:31.647 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccess (70 ms) 2024-03-07T03:27:31.647 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccess 2024-03-07T03:27:31.647 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:31.716 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccess (70 ms) 2024-03-07T03:27:31.716 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned 2024-03-07T03:27:31.717 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:31.872 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned (155 ms) 2024-03-07T03:27:31.872 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned 2024-03-07T03:27:31.872 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:31.995 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned (123 ms) 2024-03-07T03:27:31.995 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLarge 2024-03-07T03:27:31.995 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:32.123 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLarge (128 ms) 2024-03-07T03:27:32.123 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLarge 2024-03-07T03:27:32.123 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:32.250 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLarge (127 ms) 2024-03-07T03:27:32.250 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess 2024-03-07T03:27:32.250 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:32.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:32 smithi145 bash[79878]: cluster 2024-03-07T03:27:30.980040+0000 mgr.y (mgr.34107) 470 : cluster 0 pgmap v772: 457 pgs: 10 creating+peering, 8 unknown, 439 active+clean; 199 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 235 KiB/s rd, 118 KiB/s wr, 472 op/s 2024-03-07T03:27:32.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:32 smithi145 bash[79878]: cluster 2024-03-07T03:27:31.267546+0000 mon.a (mon.0) 1157 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:27:32.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:32 smithi145 bash[79878]: cluster 2024-03-07T03:27:31.288737+0000 mon.a (mon.0) 1158 : cluster 0 osdmap e678: 8 total, 8 up, 8 in 2024-03-07T03:27:32.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:32 smithi145 bash[82329]: cluster 2024-03-07T03:27:30.980040+0000 mgr.y (mgr.34107) 470 : cluster 0 pgmap v772: 457 pgs: 10 creating+peering, 8 unknown, 439 active+clean; 199 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 235 KiB/s rd, 118 KiB/s wr, 472 op/s 2024-03-07T03:27:32.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:32 smithi145 bash[82329]: cluster 2024-03-07T03:27:31.267546+0000 mon.a (mon.0) 1157 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:27:32.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:32 smithi145 bash[82329]: cluster 2024-03-07T03:27:31.288737+0000 mon.a (mon.0) 1158 : cluster 0 osdmap e678: 8 total, 8 up, 8 in 2024-03-07T03:27:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:27:32 smithi189 bash[59814]: ts=2024-03-07T03:27:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:27:32.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:32 smithi189 bash[66969]: cluster 2024-03-07T03:27:30.980040+0000 mgr.y (mgr.34107) 470 : cluster 0 pgmap v772: 457 pgs: 10 creating+peering, 8 unknown, 439 active+clean; 199 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 235 KiB/s rd, 118 KiB/s wr, 472 op/s 2024-03-07T03:27:32.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:32 smithi189 bash[66969]: cluster 2024-03-07T03:27:31.267546+0000 mon.a (mon.0) 1157 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:27:32.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:32 smithi189 bash[66969]: cluster 2024-03-07T03:27:31.288737+0000 mon.a (mon.0) 1158 : cluster 0 osdmap e678: 8 total, 8 up, 8 in 2024-03-07T03:27:32.814 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess (564 ms) 2024-03-07T03:27:32.814 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess 2024-03-07T03:27:32.814 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:33.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:27:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:27:32] "GET /metrics HTTP/1.1" 200 63036 "" "Prometheus/2.43.0" 2024-03-07T03:27:33.220 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess (406 ms) 2024-03-07T03:27:33.220 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers 2024-03-07T03:27:33.220 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:33.304 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers (84 ms) 2024-03-07T03:27:33.304 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVMismatch 2024-03-07T03:27:33.304 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:33.370 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVMismatch (66 ms) 2024-03-07T03:27:33.370 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVSuccess 2024-03-07T03:27:33.370 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:33.434 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVSuccess (65 ms) 2024-03-07T03:27:33.435 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestScatterGatherIO 2024-03-07T03:27:33.435 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:33.495 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestScatterGatherIO (60 ms) 2024-03-07T03:27:33.495 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestEmptyDiscard 2024-03-07T03:27:33.495 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:33.546 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.546 INFO:tasks.workunit.client.0.smithi145.stdout:aio discard: 0~1048576 = 0 2024-03-07T03:27:33.546 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.546 INFO:tasks.workunit.client.0.smithi145.stdout:aio discard: 0~4194304 = 0 2024-03-07T03:27:33.547 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.547 INFO:tasks.workunit.client.0.smithi145.stdout:aio discard: 3145728~1048576 = 0 2024-03-07T03:27:33.547 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestEmptyDiscard (51 ms) 2024-03-07T03:27:33.547 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestFUA 2024-03-07T03:27:33.547 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:33.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:33 smithi145 bash[79878]: audit 2024-03-07T03:27:32.650535+0000 mon.a (mon.0) 1159 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:33.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:33 smithi145 bash[82329]: audit 2024-03-07T03:27:32.650535+0000 mon.a (mon.0) 1159 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:33.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:33 smithi189 bash[66969]: audit 2024-03-07T03:27:32.650535+0000 mon.a (mon.0) 1159 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:33.837 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:33.837 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:33.837 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:33.837 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:33.837 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:33.837 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:33.838 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:33.838 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:33.838 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:33.838 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:33.838 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.838 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:33.838 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.838 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.838 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:33.838 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.838 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:33.839 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.839 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.839 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:33.839 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.839 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:33.839 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.839 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.839 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:33.839 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.839 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:33.839 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.839 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.840 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:33.840 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.840 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:33.840 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.840 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.840 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:33.840 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.840 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:33.840 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:33.840 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:33.840 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.841 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:33.841 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:33.841 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:33.841 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.841 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:33.841 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:33.841 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:33.841 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.841 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:33.841 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:33.841 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:33.841 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.842 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:33.842 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:33.842 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:33.842 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestFUA (291 ms) 2024-03-07T03:27:33.842 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestIOPP 2024-03-07T03:27:33.842 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:33.887 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:33.887 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:33.887 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:33.887 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:33.887 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:33.887 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.888 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:33.888 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.888 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.888 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:33.888 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.888 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:33.888 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.888 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.888 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:33.888 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.889 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:33.889 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.889 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.889 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:33.889 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.889 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:33.889 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.889 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.889 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:33.889 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.889 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:33.889 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.924 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.924 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:33.924 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:27:33.924 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:27:33.924 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:27:33.924 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:27:33.924 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:27:33.924 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:27:33.924 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:27:33.924 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:27:33.924 INFO:tasks.workunit.client.0.smithi145.stdout:start compare and write 2024-03-07T03:27:33.925 INFO:tasks.workunit.client.0.smithi145.stdout:compare and wrote: 512 2024-03-07T03:27:33.925 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.925 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:27:33.925 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.926 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.926 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:27:33.926 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.926 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:27:33.927 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.929 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.929 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:27:33.929 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.929 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:27:33.929 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.932 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.932 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:27:33.932 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.932 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:27:33.932 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.934 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.934 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:27:33.934 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.935 INFO:tasks.workunit.client.0.smithi145.stdout:started aio compare and write 2024-03-07T03:27:33.935 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:33.940 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:33.940 INFO:tasks.workunit.client.0.smithi145.stdout:finished aio compare and write 2024-03-07T03:27:33.940 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:33.941 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:33.941 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:33.941 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:33.941 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:33.941 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.941 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:33.941 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:33.941 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:33.942 INFO:tasks.workunit.client.0.smithi145.stdout:finished read 2024-03-07T03:27:33.942 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.942 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:33.942 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:33.943 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:33.943 INFO:tasks.workunit.client.0.smithi145.stdout:finished read 2024-03-07T03:27:33.943 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.943 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:33.943 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:33.944 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:33.944 INFO:tasks.workunit.client.0.smithi145.stdout:finished read 2024-03-07T03:27:33.944 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.944 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:33.944 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:33.945 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:33.945 INFO:tasks.workunit.client.0.smithi145.stdout:finished read 2024-03-07T03:27:33.945 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:33.945 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:33.945 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:33.946 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:33.946 INFO:tasks.workunit.client.0.smithi145.stdout:finished read 2024-03-07T03:27:33.946 INFO:tasks.workunit.client.0.smithi145.stdout:discard: 512~512 2024-03-07T03:27:33.946 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:34.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:34 smithi145 bash[79878]: cluster 2024-03-07T03:27:32.981920+0000 mgr.y (mgr.34107) 471 : cluster 0 pgmap v774: 457 pgs: 10 creating+peering, 447 active+clean; 199 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 302 KiB/s rd, 150 KiB/s wr, 605 op/s 2024-03-07T03:27:34.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:34 smithi145 bash[82329]: cluster 2024-03-07T03:27:32.981920+0000 mgr.y (mgr.34107) 471 : cluster 0 pgmap v774: 457 pgs: 10 creating+peering, 447 active+clean; 199 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 302 KiB/s rd, 150 KiB/s wr, 605 op/s 2024-03-07T03:27:34.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:34 smithi189 bash[66969]: cluster 2024-03-07T03:27:32.981920+0000 mgr.y (mgr.34107) 471 : cluster 0 pgmap v774: 457 pgs: 10 creating+peering, 447 active+clean; 199 MiB data, 961 MiB used, 714 GiB / 715 GiB avail; 302 KiB/s rd, 150 KiB/s wr, 605 op/s 2024-03-07T03:27:36.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:36 smithi145 bash[79878]: cluster 2024-03-07T03:27:34.984578+0000 mgr.y (mgr.34107) 472 : cluster 0 pgmap v775: 457 pgs: 457 active+clean; 231 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 6.7 MiB/s wr, 104 op/s 2024-03-07T03:27:36.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:36 smithi145 bash[82329]: cluster 2024-03-07T03:27:34.984578+0000 mgr.y (mgr.34107) 472 : cluster 0 pgmap v775: 457 pgs: 457 active+clean; 231 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 6.7 MiB/s wr, 104 op/s 2024-03-07T03:27:36.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:36 smithi189 bash[66969]: cluster 2024-03-07T03:27:34.984578+0000 mgr.y (mgr.34107) 472 : cluster 0 pgmap v775: 457 pgs: 457 active+clean; 231 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 6.7 MiB/s wr, 104 op/s 2024-03-07T03:27:37.769 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:37.770 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:37.770 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:37.770 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:37.770 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:37.770 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 0 2024-03-07T03:27:37.770 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.770 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.770 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 0 2024-03-07T03:27:37.770 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.770 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.770 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 16384 2024-03-07T03:27:37.771 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.771 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.771 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 16384 2024-03-07T03:27:37.771 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.771 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.771 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:37.771 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:37.771 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 49152 2024-03-07T03:27:37.771 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.771 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.771 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 49152 2024-03-07T03:27:37.772 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.772 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.772 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 65536 2024-03-07T03:27:37.772 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.772 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.772 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 65536 2024-03-07T03:27:37.772 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.772 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.772 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:37.772 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:37.772 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 98304 2024-03-07T03:27:37.772 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.772 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.773 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 98304 2024-03-07T03:27:37.773 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.773 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.773 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 114688 2024-03-07T03:27:37.773 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.773 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.773 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 114688 2024-03-07T03:27:37.773 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.773 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.773 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:37.773 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:37.773 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 147456 2024-03-07T03:27:37.774 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.774 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.774 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 147456 2024-03-07T03:27:37.774 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.774 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.774 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 163840 2024-03-07T03:27:37.774 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.774 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.774 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 163840 2024-03-07T03:27:37.774 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.774 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.774 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:37.775 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:37.775 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 196608 2024-03-07T03:27:37.775 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.775 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.775 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 196608 2024-03-07T03:27:37.775 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.775 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.775 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 212992 2024-03-07T03:27:37.775 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.775 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.775 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 212992 2024-03-07T03:27:37.776 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.776 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.776 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:37.776 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:37.776 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:37.776 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:37.776 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:37.776 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:37.776 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:37.776 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.776 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.776 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:37.776 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:37.777 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:37.777 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:37.777 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:37.777 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.777 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.777 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:37.777 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:37.777 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:37.805 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:37.805 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:37.805 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.805 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.805 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:37.805 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:37.805 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:37.835 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:37.835 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:37.835 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.835 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.835 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:37.835 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:37.835 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:37.836 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:37.836 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:37.836 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:37.836 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:37.836 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:37.836 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:37.927 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:37.927 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:37.927 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.928 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.928 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:37.928 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:37.928 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:37.986 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:37.986 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:37.987 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:37.987 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:37.987 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:37.987 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:37.987 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:38.073 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:38.074 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:38.074 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:38.074 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:38.074 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:38.074 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:38.074 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:38.191 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:38.192 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:38.192 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:38.192 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:38.192 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:38.192 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:38.192 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:38.192 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:38.192 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:38.192 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:38.192 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:38.192 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:38.192 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:38.396 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:38.396 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:38.396 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:38.396 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:38.396 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:38.396 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:38.396 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:38.565 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:38.566 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:38.566 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:38.566 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:38.566 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:38.566 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:38.566 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:38.768 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:38.768 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:38.768 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:38.768 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:38.768 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:38.768 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:38.769 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:38.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:38 smithi145 bash[79878]: cluster 2024-03-07T03:27:36.987255+0000 mgr.y (mgr.34107) 473 : cluster 0 pgmap v776: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 7.3 MiB/s wr, 172 op/s 2024-03-07T03:27:38.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:38 smithi145 bash[79878]: cluster 2024-03-07T03:27:37.461374+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e679: 8 total, 8 up, 8 in 2024-03-07T03:27:38.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:38 smithi145 bash[82329]: cluster 2024-03-07T03:27:36.987255+0000 mgr.y (mgr.34107) 473 : cluster 0 pgmap v776: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 7.3 MiB/s wr, 172 op/s 2024-03-07T03:27:38.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:38 smithi145 bash[82329]: cluster 2024-03-07T03:27:37.461374+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e679: 8 total, 8 up, 8 in 2024-03-07T03:27:38.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:38 smithi189 bash[66969]: cluster 2024-03-07T03:27:36.987255+0000 mgr.y (mgr.34107) 473 : cluster 0 pgmap v776: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 7.3 MiB/s wr, 172 op/s 2024-03-07T03:27:38.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:38 smithi189 bash[66969]: cluster 2024-03-07T03:27:37.461374+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e679: 8 total, 8 up, 8 in 2024-03-07T03:27:38.952 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:38.952 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:38.952 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:38.952 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:38.952 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:38.952 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:38.952 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:38.952 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:38.953 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:38.953 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:38.953 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:38.953 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:38.953 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:39.222 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:39.222 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:39.222 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:39.222 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:39.223 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:39.223 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:39.223 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:39.481 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:39.481 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:39.481 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:39.481 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:39.481 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:39.481 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:39.481 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:39.794 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:39.794 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:39.794 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:39.794 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:39.794 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:39.794 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:39.795 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:40.111 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:40.111 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:40.111 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:40.111 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:40.111 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:40.111 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:40.111 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:40.111 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:40.111 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:40.111 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:40.111 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:40.111 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:40.111 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:40.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:40 smithi189 bash[66969]: cluster 2024-03-07T03:27:38.989499+0000 mgr.y (mgr.34107) 474 : cluster 0 pgmap v778: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.2 MiB/s wr, 832 op/s 2024-03-07T03:27:40.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:40 smithi189 bash[66969]: audit 2024-03-07T03:27:39.083442+0000 mon.a (mon.0) 1161 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:40.518 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:40.518 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:40.518 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:40.518 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:40.518 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:40.518 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:40.518 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:40.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:40 smithi145 bash[79878]: cluster 2024-03-07T03:27:38.989499+0000 mgr.y (mgr.34107) 474 : cluster 0 pgmap v778: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.2 MiB/s wr, 832 op/s 2024-03-07T03:27:40.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:40 smithi145 bash[79878]: audit 2024-03-07T03:27:39.083442+0000 mon.a (mon.0) 1161 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:40.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:40 smithi145 bash[82329]: cluster 2024-03-07T03:27:38.989499+0000 mgr.y (mgr.34107) 474 : cluster 0 pgmap v778: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.2 MiB/s wr, 832 op/s 2024-03-07T03:27:40.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:40 smithi145 bash[82329]: audit 2024-03-07T03:27:39.083442+0000 mon.a (mon.0) 1161 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:27:40.874 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:40.875 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:40.875 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:40.875 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:40.875 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:40.875 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:40.875 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:41.265 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:41.265 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:41.265 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:41.265 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:41.265 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:41.265 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:41.265 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:41.638 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:41.638 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:41.638 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:41.638 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:41.638 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:41.639 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:41.639 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:41.639 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:41.639 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:41.639 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:41.639 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestIOPP (7801 ms) 2024-03-07T03:27:41.639 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP 2024-03-07T03:27:41.639 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:41.705 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP (67 ms) 2024-03-07T03:27:41.705 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP 2024-03-07T03:27:41.705 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:41.771 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP (67 ms) 2024-03-07T03:27:41.772 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP 2024-03-07T03:27:41.772 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:41.838 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP (66 ms) 2024-03-07T03:27:41.838 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP 2024-03-07T03:27:41.838 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:41.903 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP (65 ms) 2024-03-07T03:27:41.904 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchPP 2024-03-07T03:27:41.904 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:41.978 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchPP (75 ms) 2024-03-07T03:27:41.978 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchPP 2024-03-07T03:27:41.978 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:42.040 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:42.051 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchPP (73 ms) 2024-03-07T03:27:42.051 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP 2024-03-07T03:27:42.051 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:42.123 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP (72 ms) 2024-03-07T03:27:42.124 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP 2024-03-07T03:27:42.124 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:42.186 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:42.197 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP (73 ms) 2024-03-07T03:27:42.197 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessPP 2024-03-07T03:27:42.197 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:42.267 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessPP (70 ms) 2024-03-07T03:27:42.267 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessPP 2024-03-07T03:27:42.267 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:42.324 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:42.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:42 smithi189 bash[66969]: cluster 2024-03-07T03:27:40.991683+0000 mgr.y (mgr.34107) 475 : cluster 0 pgmap v779: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.9 MiB/s wr, 686 op/s 2024-03-07T03:27:42.329 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:27:42 smithi189 bash[59814]: ts=2024-03-07T03:27:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:27:42.334 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessPP (67 ms) 2024-03-07T03:27:42.334 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP 2024-03-07T03:27:42.334 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:42.406 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP (72 ms) 2024-03-07T03:27:42.406 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP 2024-03-07T03:27:42.406 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:42.466 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:42.477 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP (72 ms) 2024-03-07T03:27:42.477 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP 2024-03-07T03:27:42.478 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:42.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:42 smithi145 bash[79878]: cluster 2024-03-07T03:27:40.991683+0000 mgr.y (mgr.34107) 475 : cluster 0 pgmap v779: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.9 MiB/s wr, 686 op/s 2024-03-07T03:27:42.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:42 smithi145 bash[82329]: cluster 2024-03-07T03:27:40.991683+0000 mgr.y (mgr.34107) 475 : cluster 0 pgmap v779: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.9 MiB/s wr, 686 op/s 2024-03-07T03:27:42.628 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP (150 ms) 2024-03-07T03:27:42.628 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP 2024-03-07T03:27:42.628 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:42.719 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:42.748 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP (120 ms) 2024-03-07T03:27:42.748 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLargePP 2024-03-07T03:27:42.748 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:42.879 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLargePP (132 ms) 2024-03-07T03:27:42.880 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLargePP 2024-03-07T03:27:42.880 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:43.001 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:43.054 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLargePP (174 ms) 2024-03-07T03:27:43.054 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP 2024-03-07T03:27:43.054 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:43.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:27:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:27:42] "GET /metrics HTTP/1.1" 200 63036 "" "Prometheus/2.43.0" 2024-03-07T03:27:43.482 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP (428 ms) 2024-03-07T03:27:43.482 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP 2024-03-07T03:27:43.482 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:43.845 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:43.882 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP (400 ms) 2024-03-07T03:27:43.882 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestIOPPWithIOHint 2024-03-07T03:27:43.883 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:43.956 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:43.956 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:43.956 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:43.956 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:43.956 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:27:43.957 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:43.957 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:43.957 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:43.959 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:43.959 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:43.959 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:43.959 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:43.959 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:43.962 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:43.962 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:43.962 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:43.962 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:43.962 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:43.964 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:43.965 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:43.965 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:43.965 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:43.965 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:43.970 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:43.970 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:43.970 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:43.970 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:27:43.970 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:43.972 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:43.972 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:27:43.972 INFO:tasks.workunit.client.0.smithi145.stdout:read: 512 2024-03-07T03:27:43.972 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:43.972 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:43.972 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:43.973 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:43.973 INFO:tasks.workunit.client.0.smithi145.stdout:finished read 2024-03-07T03:27:43.973 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:43.973 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:43.973 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:43.974 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:43.974 INFO:tasks.workunit.client.0.smithi145.stdout:finished read 2024-03-07T03:27:43.974 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:43.974 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:43.974 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:43.975 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:43.975 INFO:tasks.workunit.client.0.smithi145.stdout:finished read 2024-03-07T03:27:43.975 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:43.975 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:43.975 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:43.976 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:43.976 INFO:tasks.workunit.client.0.smithi145.stdout:finished read 2024-03-07T03:27:43.976 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:43.976 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:27:43.976 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:27:44.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:44 smithi189 bash[66969]: cluster 2024-03-07T03:27:42.993931+0000 mgr.y (mgr.34107) 476 : cluster 0 pgmap v780: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 5.7 MiB/s wr, 662 op/s 2024-03-07T03:27:44.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:44 smithi145 bash[79878]: cluster 2024-03-07T03:27:42.993931+0000 mgr.y (mgr.34107) 476 : cluster 0 pgmap v780: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 5.7 MiB/s wr, 662 op/s 2024-03-07T03:27:44.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:44 smithi145 bash[82329]: cluster 2024-03-07T03:27:42.993931+0000 mgr.y (mgr.34107) 476 : cluster 0 pgmap v780: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 5.7 MiB/s wr, 662 op/s 2024-03-07T03:27:46.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:46 smithi145 bash[79878]: cluster 2024-03-07T03:27:44.996417+0000 mgr.y (mgr.34107) 477 : cluster 0 pgmap v781: 457 pgs: 457 active+clean; 269 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 4.3 MiB/s wr, 1.08k op/s 2024-03-07T03:27:46.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:46 smithi145 bash[82329]: cluster 2024-03-07T03:27:44.996417+0000 mgr.y (mgr.34107) 477 : cluster 0 pgmap v781: 457 pgs: 457 active+clean; 269 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 4.3 MiB/s wr, 1.08k op/s 2024-03-07T03:27:46.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:46 smithi189 bash[66969]: cluster 2024-03-07T03:27:44.996417+0000 mgr.y (mgr.34107) 477 : cluster 0 pgmap v781: 457 pgs: 457 active+clean; 269 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 4.3 MiB/s wr, 1.08k op/s 2024-03-07T03:27:48.036 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:27:48.037 INFO:tasks.workunit.client.0.smithi145.stdout:finished read 2024-03-07T03:27:48.037 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 0 2024-03-07T03:27:48.037 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.037 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.037 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 0 2024-03-07T03:27:48.037 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.037 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.037 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 16384 2024-03-07T03:27:48.037 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.037 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.037 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 16384 2024-03-07T03:27:48.037 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.037 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 49152 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 49152 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 65536 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 65536 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.038 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 98304 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 98304 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 114688 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 114688 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.039 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 147456 2024-03-07T03:27:48.040 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.040 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.040 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 147456 2024-03-07T03:27:48.040 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.040 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.040 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 163840 2024-03-07T03:27:48.040 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.040 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.040 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 163840 2024-03-07T03:27:48.040 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.040 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.040 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.040 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 196608 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 196608 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 212992 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 212992 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.041 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.042 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.042 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.042 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:48.042 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:48.042 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:48.042 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.042 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.042 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.042 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.042 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:48.043 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:48.043 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:48.043 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.043 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.043 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.043 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.043 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:48.075 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:48.075 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:48.075 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.075 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.075 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.075 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.076 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:48.114 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:48.114 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:48.114 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.114 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.114 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.114 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.114 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.114 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.114 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.114 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.115 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.115 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.115 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:48.207 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:48.207 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:48.207 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.207 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.207 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.207 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.207 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:48.303 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:48.303 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:48.303 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.303 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.303 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.303 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.303 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:48.422 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:48.422 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:48.422 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.422 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.423 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.423 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.423 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:48.546 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:48.546 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:48.546 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.546 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.546 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.546 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.546 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.546 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.546 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.547 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:48.547 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.547 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.547 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:48.734 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:48.734 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:48.734 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.734 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.734 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.734 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.734 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:48.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:48 smithi145 bash[79878]: cluster 2024-03-07T03:27:46.999097+0000 mgr.y (mgr.34107) 478 : cluster 0 pgmap v782: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.07k op/s 2024-03-07T03:27:48.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:48 smithi145 bash[79878]: audit 2024-03-07T03:27:47.651176+0000 mon.a (mon.0) 1162 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:48.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:48 smithi145 bash[82329]: cluster 2024-03-07T03:27:46.999097+0000 mgr.y (mgr.34107) 478 : cluster 0 pgmap v782: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.07k op/s 2024-03-07T03:27:48.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:48 smithi145 bash[82329]: audit 2024-03-07T03:27:47.651176+0000 mon.a (mon.0) 1162 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:48.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:48 smithi189 bash[66969]: cluster 2024-03-07T03:27:46.999097+0000 mgr.y (mgr.34107) 478 : cluster 0 pgmap v782: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.07k op/s 2024-03-07T03:27:48.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:48 smithi189 bash[66969]: audit 2024-03-07T03:27:47.651176+0000 mon.a (mon.0) 1162 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:48.886 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:48.886 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:48.886 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:48.887 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:48.887 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:48.887 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:48.887 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:49.125 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:49.126 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:49.126 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:49.126 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:49.126 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:49.126 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:49.126 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:49.342 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:49.343 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:49.343 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:49.343 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:49.343 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:49.343 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:49.343 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:49.343 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:49.343 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:49.343 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:49.343 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:49.343 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:49.343 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:49.612 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:49.612 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:49.612 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:49.613 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:49.613 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:49.613 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:49.613 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:49.887 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:49.887 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:49.887 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:49.887 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:49.887 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:49.887 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:49.887 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:50.167 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:50.168 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:50.168 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:50.168 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:50.168 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:50.168 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:50.168 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:50.436 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:50.436 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:50.437 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:50.437 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:50.437 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:50.437 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:50.437 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:50.437 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:50.437 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:50.437 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:50.437 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:50.437 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:50.437 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:50.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:50 smithi145 bash[79878]: cluster 2024-03-07T03:27:49.001287+0000 mgr.y (mgr.34107) 479 : cluster 0 pgmap v783: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 5.1 MiB/s wr, 927 op/s 2024-03-07T03:27:50.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:50 smithi145 bash[82329]: cluster 2024-03-07T03:27:49.001287+0000 mgr.y (mgr.34107) 479 : cluster 0 pgmap v783: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 5.1 MiB/s wr, 927 op/s 2024-03-07T03:27:50.811 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:50.811 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:50.811 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:50.811 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:50.811 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:50.811 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:50.811 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:50.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:50 smithi189 bash[66969]: cluster 2024-03-07T03:27:49.001287+0000 mgr.y (mgr.34107) 479 : cluster 0 pgmap v783: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 5.1 MiB/s wr, 927 op/s 2024-03-07T03:27:51.142 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:51.142 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:51.143 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:51.143 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:51.143 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:51.143 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:51.143 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:51.522 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:51.522 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:51.522 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:51.522 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:51.523 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:51.523 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:51.523 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:27:51.928 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:27:51.928 INFO:tasks.workunit.client.0.smithi145.stdout:finished writesame 2024-03-07T03:27:51.928 INFO:tasks.workunit.client.0.smithi145.stdout:to verify the data 2024-03-07T03:27:51.928 INFO:tasks.workunit.client.0.smithi145.stdout:verified 2024-03-07T03:27:51.928 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:51.928 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:51.928 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:51.928 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:27:51.928 INFO:tasks.workunit.client.0.smithi145.stdout:started writesame 2024-03-07T03:27:51.929 INFO:tasks.workunit.client.0.smithi145.stdout:expected fail, finished writesame 2024-03-07T03:27:51.929 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestIOPPWithIOHint (8046 ms) 2024-03-07T03:27:51.929 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestIOToSnapshot 2024-03-07T03:27:51.929 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:52.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:52 smithi189 bash[66969]: cluster 2024-03-07T03:27:51.003491+0000 mgr.y (mgr.34107) 480 : cluster 0 pgmap v784: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 4.9 MiB/s wr, 877 op/s 2024-03-07T03:27:52.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:27:52 smithi189 bash[59814]: ts=2024-03-07T03:27:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:27:52.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:52 smithi145 bash[79878]: cluster 2024-03-07T03:27:51.003491+0000 mgr.y (mgr.34107) 480 : cluster 0 pgmap v784: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 4.9 MiB/s wr, 877 op/s 2024-03-07T03:27:52.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:52 smithi145 bash[82329]: cluster 2024-03-07T03:27:51.003491+0000 mgr.y (mgr.34107) 480 : cluster 0 pgmap v784: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 4.9 MiB/s wr, 877 op/s 2024-03-07T03:27:53.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:27:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:27:52] "GET /metrics HTTP/1.1" 200 63055 "" "Prometheus/2.43.0" 2024-03-07T03:27:53.557 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 0 2024-03-07T03:27:53.557 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 0 2024-03-07T03:27:53.557 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 1 2024-03-07T03:27:53.557 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:27:53.557 INFO:tasks.workunit.client.0.smithi145.stdout:snap: orig 2024-03-07T03:27:53.557 INFO:tasks.workunit.client.0.smithi145.stdout:found orig with size 2097152 2024-03-07T03:27:53.557 INFO:tasks.workunit.client.0.smithi145.stdout:read: 80 2024-03-07T03:27:53.557 INFO:tasks.workunit.client.0.smithi145.stdout:write test data! 2024-03-07T03:27:53.558 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 80 2024-03-07T03:27:53.558 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 2 2024-03-07T03:27:53.558 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 2 2024-03-07T03:27:53.558 INFO:tasks.workunit.client.0.smithi145.stdout:snap: orig 2024-03-07T03:27:53.558 INFO:tasks.workunit.client.0.smithi145.stdout:snap: written 2024-03-07T03:27:53.558 INFO:tasks.workunit.client.0.smithi145.stdout:found orig with size 2097152 2024-03-07T03:27:53.558 INFO:tasks.workunit.client.0.smithi145.stdout:found written with size 2097152 2024-03-07T03:27:53.558 INFO:tasks.workunit.client.0.smithi145.stdout:read: 80 2024-03-07T03:27:53.558 INFO:tasks.workunit.client.0.smithi145.stdout:read: 80 2024-03-07T03:27:53.558 INFO:tasks.workunit.client.0.smithi145.stdout:read: 80 2024-03-07T03:27:53.558 INFO:tasks.workunit.client.0.smithi145.stdout:write to snapshot returned -30 2024-03-07T03:27:53.558 INFO:tasks.workunit.client.0.smithi145.stdout:Read-only file system 2024-03-07T03:27:53.627 INFO:tasks.workunit.client.0.smithi145.stdout:read: 80 2024-03-07T03:27:53.627 INFO:tasks.workunit.client.0.smithi145.stdout:read: 80 2024-03-07T03:27:53.627 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 80 2024-03-07T03:27:53.627 INFO:tasks.workunit.client.0.smithi145.stdout:opening testimg@orig 2024-03-07T03:27:53.627 INFO:tasks.workunit.client.0.smithi145.stdout:read: 80 2024-03-07T03:27:53.627 INFO:tasks.workunit.client.0.smithi145.stdout:write to snapshot returned -30 2024-03-07T03:27:53.627 INFO:tasks.workunit.client.0.smithi145.stdout:Read-only file system 2024-03-07T03:27:53.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:53 smithi145 bash[79878]: cluster 2024-03-07T03:27:52.483550+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e680: 8 total, 8 up, 8 in 2024-03-07T03:27:53.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:53 smithi145 bash[82329]: cluster 2024-03-07T03:27:52.483550+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e680: 8 total, 8 up, 8 in 2024-03-07T03:27:53.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:53 smithi189 bash[66969]: cluster 2024-03-07T03:27:52.483550+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e680: 8 total, 8 up, 8 in 2024-03-07T03:27:54.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:54 smithi145 bash[79878]: cluster 2024-03-07T03:27:53.004465+0000 mgr.y (mgr.34107) 481 : cluster 0 pgmap v786: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.05k op/s 2024-03-07T03:27:54.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:54 smithi145 bash[79878]: cluster 2024-03-07T03:27:53.496879+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e681: 8 total, 8 up, 8 in 2024-03-07T03:27:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:54 smithi145 bash[82329]: cluster 2024-03-07T03:27:53.004465+0000 mgr.y (mgr.34107) 481 : cluster 0 pgmap v786: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.05k op/s 2024-03-07T03:27:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:54 smithi145 bash[82329]: cluster 2024-03-07T03:27:53.496879+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e681: 8 total, 8 up, 8 in 2024-03-07T03:27:54.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:54 smithi189 bash[66969]: cluster 2024-03-07T03:27:53.004465+0000 mgr.y (mgr.34107) 481 : cluster 0 pgmap v786: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1.05k op/s 2024-03-07T03:27:54.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:54 smithi189 bash[66969]: cluster 2024-03-07T03:27:53.496879+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e681: 8 total, 8 up, 8 in 2024-03-07T03:27:55.570 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 2 2024-03-07T03:27:55.571 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 2 2024-03-07T03:27:55.571 INFO:tasks.workunit.client.0.smithi145.stdout:snap: orig 2024-03-07T03:27:55.571 INFO:tasks.workunit.client.0.smithi145.stdout:snap: written 2024-03-07T03:27:55.571 INFO:tasks.workunit.client.0.smithi145.stdout:found orig with size 2097152 2024-03-07T03:27:55.571 INFO:tasks.workunit.client.0.smithi145.stdout:found written with size 2097152 2024-03-07T03:27:55.571 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 1 2024-03-07T03:27:55.571 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:27:55.571 INFO:tasks.workunit.client.0.smithi145.stdout:snap: orig 2024-03-07T03:27:55.571 INFO:tasks.workunit.client.0.smithi145.stdout:found orig with size 2097152 2024-03-07T03:27:55.571 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 0 2024-03-07T03:27:55.571 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 0 2024-03-07T03:27:55.571 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestIOToSnapshot (3642 ms) 2024-03-07T03:27:55.571 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestSnapshotDeletedIo 2024-03-07T03:27:55.572 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:55.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:55 smithi145 bash[79878]: cluster 2024-03-07T03:27:54.506611+0000 mon.a (mon.0) 1165 : cluster 0 osdmap e682: 8 total, 8 up, 8 in 2024-03-07T03:27:55.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:55 smithi145 bash[82329]: cluster 2024-03-07T03:27:54.506611+0000 mon.a (mon.0) 1165 : cluster 0 osdmap e682: 8 total, 8 up, 8 in 2024-03-07T03:27:55.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:55 smithi189 bash[66969]: cluster 2024-03-07T03:27:54.506611+0000 mon.a (mon.0) 1165 : cluster 0 osdmap e682: 8 total, 8 up, 8 in 2024-03-07T03:27:56.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:56 smithi145 bash[79878]: cluster 2024-03-07T03:27:55.005891+0000 mgr.y (mgr.34107) 482 : cluster 0 pgmap v789: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 626 KiB/s rd, 293 KiB/s wr, 1.26k op/s 2024-03-07T03:27:56.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:56 smithi145 bash[79878]: cluster 2024-03-07T03:27:55.516483+0000 mon.a (mon.0) 1166 : cluster 0 osdmap e683: 8 total, 8 up, 8 in 2024-03-07T03:27:56.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:56 smithi145 bash[82329]: cluster 2024-03-07T03:27:55.005891+0000 mgr.y (mgr.34107) 482 : cluster 0 pgmap v789: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 626 KiB/s rd, 293 KiB/s wr, 1.26k op/s 2024-03-07T03:27:56.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:56 smithi145 bash[82329]: cluster 2024-03-07T03:27:55.516483+0000 mon.a (mon.0) 1166 : cluster 0 osdmap e683: 8 total, 8 up, 8 in 2024-03-07T03:27:56.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:56 smithi189 bash[66969]: cluster 2024-03-07T03:27:55.005891+0000 mgr.y (mgr.34107) 482 : cluster 0 pgmap v789: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 626 KiB/s rd, 293 KiB/s wr, 1.26k op/s 2024-03-07T03:27:56.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:56 smithi189 bash[66969]: cluster 2024-03-07T03:27:55.516483+0000 mon.a (mon.0) 1166 : cluster 0 osdmap e683: 8 total, 8 up, 8 in 2024-03-07T03:27:57.584 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestSnapshotDeletedIo (2014 ms) 2024-03-07T03:27:57.584 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestClone 2024-03-07T03:27:57.584 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:57.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:57 smithi145 bash[79878]: cluster 2024-03-07T03:27:56.525535+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e684: 8 total, 8 up, 8 in 2024-03-07T03:27:57.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:57 smithi145 bash[82329]: cluster 2024-03-07T03:27:56.525535+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e684: 8 total, 8 up, 8 in 2024-03-07T03:27:57.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:57 smithi189 bash[66969]: cluster 2024-03-07T03:27:56.525535+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e684: 8 total, 8 up, 8 in 2024-03-07T03:27:58.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:58 smithi145 bash[79878]: cluster 2024-03-07T03:27:57.007879+0000 mgr.y (mgr.34107) 483 : cluster 0 pgmap v792: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 311 KiB/s rd, 106 KiB/s wr, 615 op/s 2024-03-07T03:27:58.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:58 smithi145 bash[79878]: cluster 2024-03-07T03:27:57.536182+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e685: 8 total, 8 up, 8 in 2024-03-07T03:27:58.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:58 smithi145 bash[82329]: cluster 2024-03-07T03:27:57.007879+0000 mgr.y (mgr.34107) 483 : cluster 0 pgmap v792: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 311 KiB/s rd, 106 KiB/s wr, 615 op/s 2024-03-07T03:27:58.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:58 smithi145 bash[82329]: cluster 2024-03-07T03:27:57.536182+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e685: 8 total, 8 up, 8 in 2024-03-07T03:27:58.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:58 smithi189 bash[66969]: cluster 2024-03-07T03:27:57.007879+0000 mgr.y (mgr.34107) 483 : cluster 0 pgmap v792: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 311 KiB/s rd, 106 KiB/s wr, 615 op/s 2024-03-07T03:27:58.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:58 smithi189 bash[66969]: cluster 2024-03-07T03:27:57.536182+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e685: 8 total, 8 up, 8 in 2024-03-07T03:27:59.589 INFO:tasks.workunit.client.0.smithi145.stdout:made parent image "parent" 2024-03-07T03:27:59.589 INFO:tasks.workunit.client.0.smithi145.stdout:parent has no parent info 2024-03-07T03:27:59.589 INFO:tasks.workunit.client.0.smithi145.stdout:made snapshot "parent@parent_snap" 2024-03-07T03:27:59.589 INFO:tasks.workunit.client.0.smithi145.stdout:can't unprotect an unprotected snap 2024-03-07T03:27:59.589 INFO:tasks.workunit.client.0.smithi145.stdout:can't protect a protected snap 2024-03-07T03:27:59.589 INFO:tasks.workunit.client.0.smithi145.stdout:made and opened clone "child" 2024-03-07T03:27:59.589 INFO:tasks.workunit.client.0.smithi145.stdout:read: 8 2024-03-07T03:27:59.589 INFO:tasks.workunit.client.0.smithi145.stdout:read: 8 2024-03-07T03:27:59.589 INFO:tasks.workunit.client.0.smithi145.stdout:read: 8 2024-03-07T03:27:59.589 INFO:tasks.workunit.client.0.smithi145.stdout:sizes and overlaps are good between parent and child 2024-03-07T03:27:59.589 INFO:tasks.workunit.client.0.smithi145.stdout:child image successfully cloned all image-meta pairs 2024-03-07T03:27:59.589 INFO:tasks.workunit.client.0.smithi145.stdout:sized down clone, changed overlap 2024-03-07T03:27:59.589 INFO:tasks.workunit.client.0.smithi145.stdout:parent info: size 4194304 obj_size 4194304 parent_pool 18446744073709551615 2024-03-07T03:27:59.590 INFO:tasks.workunit.client.0.smithi145.stdout:sized up clone, changed size but not overlap or parent's size 2024-03-07T03:27:59.590 INFO:tasks.workunit.client.0.smithi145.stdout:can't remove parent while child still exists 2024-03-07T03:27:59.590 INFO:tasks.workunit.client.0.smithi145.stdout:can't remove parent while still protected 2024-03-07T03:27:59.590 INFO:tasks.workunit.client.0.smithi145.stdout:removed parent snap after unprotecting 2024-03-07T03:27:59.590 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestClone (2004 ms) 2024-03-07T03:27:59.590 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestClone2 2024-03-07T03:27:59.590 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:27:59.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:27:59 smithi145 bash[79878]: cluster 2024-03-07T03:27:58.545330+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e686: 8 total, 8 up, 8 in 2024-03-07T03:27:59.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:27:59 smithi145 bash[82329]: cluster 2024-03-07T03:27:58.545330+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e686: 8 total, 8 up, 8 in 2024-03-07T03:27:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:27:59 smithi189 bash[66969]: cluster 2024-03-07T03:27:58.545330+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e686: 8 total, 8 up, 8 in 2024-03-07T03:28:00.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:00 smithi145 bash[79878]: cluster 2024-03-07T03:27:59.010147+0000 mgr.y (mgr.34107) 484 : cluster 0 pgmap v795: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 7.0 KiB/s wr, 62 op/s 2024-03-07T03:28:00.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:00 smithi145 bash[79878]: cluster 2024-03-07T03:27:59.558590+0000 mon.a (mon.0) 1170 : cluster 0 osdmap e687: 8 total, 8 up, 8 in 2024-03-07T03:28:00.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:00 smithi145 bash[82329]: cluster 2024-03-07T03:27:59.010147+0000 mgr.y (mgr.34107) 484 : cluster 0 pgmap v795: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 7.0 KiB/s wr, 62 op/s 2024-03-07T03:28:00.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:00 smithi145 bash[82329]: cluster 2024-03-07T03:27:59.558590+0000 mon.a (mon.0) 1170 : cluster 0 osdmap e687: 8 total, 8 up, 8 in 2024-03-07T03:28:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:00 smithi189 bash[66969]: cluster 2024-03-07T03:27:59.010147+0000 mgr.y (mgr.34107) 484 : cluster 0 pgmap v795: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 7.0 KiB/s wr, 62 op/s 2024-03-07T03:28:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:00 smithi189 bash[66969]: cluster 2024-03-07T03:27:59.558590+0000 mon.a (mon.0) 1170 : cluster 0 osdmap e687: 8 total, 8 up, 8 in 2024-03-07T03:28:00.862 INFO:tasks.workunit.client.0.smithi145.stdout:made parent image "parent" 2024-03-07T03:28:00.862 INFO:tasks.workunit.client.0.smithi145.stdout:parent has no parent info 2024-03-07T03:28:00.862 INFO:tasks.workunit.client.0.smithi145.stdout:made snapshot "parent@parent_snap" 2024-03-07T03:28:00.862 INFO:tasks.workunit.client.0.smithi145.stdout:made and opened clone "child" 2024-03-07T03:28:00.863 INFO:tasks.workunit.client.0.smithi145.stdout:child image successfully cloned all image-meta pairs 2024-03-07T03:28:00.863 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestClone2 (1274 ms) 2024-03-07T03:28:00.863 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ListChildren 2024-03-07T03:28:01.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:01 smithi189 bash[66969]: cluster 2024-03-07T03:28:00.564459+0000 mon.a (mon.0) 1171 : cluster 0 osdmap e688: 8 total, 8 up, 8 in 2024-03-07T03:28:02.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:01 smithi145 bash[79878]: cluster 2024-03-07T03:28:00.564459+0000 mon.a (mon.0) 1171 : cluster 0 osdmap e688: 8 total, 8 up, 8 in 2024-03-07T03:28:02.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:01 smithi145 bash[82329]: cluster 2024-03-07T03:28:00.564459+0000 mon.a (mon.0) 1171 : cluster 0 osdmap e688: 8 total, 8 up, 8 in 2024-03-07T03:28:02.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:28:02 smithi189 bash[59814]: ts=2024-03-07T03:28:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:28:03.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:28:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:28:02] "GET /metrics HTTP/1.1" 200 63055 "" "Prometheus/2.43.0" 2024-03-07T03:28:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:02 smithi145 bash[79878]: cluster 2024-03-07T03:28:01.011923+0000 mgr.y (mgr.34107) 485 : cluster 0 pgmap v798: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 399 KiB/s rd, 29 KiB/s wr, 479 op/s 2024-03-07T03:28:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:02 smithi145 bash[79878]: cluster 2024-03-07T03:28:01.586770+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e689: 8 total, 8 up, 8 in 2024-03-07T03:28:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:02 smithi145 bash[79878]: audit 2024-03-07T03:28:01.591030+0000 mon.c (mon.2) 36 : audit 1 from='client.? 172.21.15.145:0/1775010251' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:02 smithi145 bash[79878]: audit 2024-03-07T03:28:01.596667+0000 mon.a (mon.0) 1173 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:02 smithi145 bash[79878]: audit 2024-03-07T03:28:02.463569+0000 mon.a (mon.0) 1174 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:02 smithi145 bash[79878]: cluster 2024-03-07T03:28:02.471060+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e690: 8 total, 8 up, 8 in 2024-03-07T03:28:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:02 smithi145 bash[82329]: cluster 2024-03-07T03:28:01.011923+0000 mgr.y (mgr.34107) 485 : cluster 0 pgmap v798: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 399 KiB/s rd, 29 KiB/s wr, 479 op/s 2024-03-07T03:28:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:02 smithi145 bash[82329]: cluster 2024-03-07T03:28:01.586770+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e689: 8 total, 8 up, 8 in 2024-03-07T03:28:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:02 smithi145 bash[82329]: audit 2024-03-07T03:28:01.591030+0000 mon.c (mon.2) 36 : audit 1 from='client.? 172.21.15.145:0/1775010251' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:02 smithi145 bash[82329]: audit 2024-03-07T03:28:01.596667+0000 mon.a (mon.0) 1173 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:02 smithi145 bash[82329]: audit 2024-03-07T03:28:02.463569+0000 mon.a (mon.0) 1174 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:02 smithi145 bash[82329]: cluster 2024-03-07T03:28:02.471060+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e690: 8 total, 8 up, 8 in 2024-03-07T03:28:03.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:02 smithi189 bash[66969]: cluster 2024-03-07T03:28:01.011923+0000 mgr.y (mgr.34107) 485 : cluster 0 pgmap v798: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 399 KiB/s rd, 29 KiB/s wr, 479 op/s 2024-03-07T03:28:03.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:02 smithi189 bash[66969]: cluster 2024-03-07T03:28:01.586770+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e689: 8 total, 8 up, 8 in 2024-03-07T03:28:03.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:02 smithi189 bash[66969]: audit 2024-03-07T03:28:01.591030+0000 mon.c (mon.2) 36 : audit 1 from='client.? 172.21.15.145:0/1775010251' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:03.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:02 smithi189 bash[66969]: audit 2024-03-07T03:28:01.596667+0000 mon.a (mon.0) 1173 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:03.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:02 smithi189 bash[66969]: audit 2024-03-07T03:28:02.463569+0000 mon.a (mon.0) 1174 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:03.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:02 smithi189 bash[66969]: cluster 2024-03-07T03:28:02.471060+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e690: 8 total, 8 up, 8 in 2024-03-07T03:28:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:03 smithi145 bash[79878]: audit 2024-03-07T03:28:02.972458+0000 mon.a (mon.0) 1176 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:03 smithi145 bash[79878]: cluster 2024-03-07T03:28:03.018398+0000 mgr.y (mgr.34107) 486 : cluster 0 pgmap v801: 489 pgs: 6 creating+peering, 26 unknown, 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 681 KiB/s rd, 47 KiB/s wr, 808 op/s 2024-03-07T03:28:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:03 smithi145 bash[79878]: cluster 2024-03-07T03:28:03.473915+0000 mon.a (mon.0) 1177 : cluster 0 osdmap e691: 8 total, 8 up, 8 in 2024-03-07T03:28:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:03 smithi145 bash[79878]: audit 2024-03-07T03:28:03.487857+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.145:0/3084881735' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:04.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:03 smithi145 bash[79878]: audit 2024-03-07T03:28:03.490265+0000 mon.a (mon.0) 1178 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:04.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:03 smithi145 bash[82329]: audit 2024-03-07T03:28:02.972458+0000 mon.a (mon.0) 1176 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:04.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:03 smithi145 bash[82329]: cluster 2024-03-07T03:28:03.018398+0000 mgr.y (mgr.34107) 486 : cluster 0 pgmap v801: 489 pgs: 6 creating+peering, 26 unknown, 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 681 KiB/s rd, 47 KiB/s wr, 808 op/s 2024-03-07T03:28:04.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:03 smithi145 bash[82329]: cluster 2024-03-07T03:28:03.473915+0000 mon.a (mon.0) 1177 : cluster 0 osdmap e691: 8 total, 8 up, 8 in 2024-03-07T03:28:04.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:03 smithi145 bash[82329]: audit 2024-03-07T03:28:03.487857+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.145:0/3084881735' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:04.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:03 smithi145 bash[82329]: audit 2024-03-07T03:28:03.490265+0000 mon.a (mon.0) 1178 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:04.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:03 smithi189 bash[66969]: audit 2024-03-07T03:28:02.972458+0000 mon.a (mon.0) 1176 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:04.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:03 smithi189 bash[66969]: cluster 2024-03-07T03:28:03.018398+0000 mgr.y (mgr.34107) 486 : cluster 0 pgmap v801: 489 pgs: 6 creating+peering, 26 unknown, 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 681 KiB/s rd, 47 KiB/s wr, 808 op/s 2024-03-07T03:28:04.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:03 smithi189 bash[66969]: cluster 2024-03-07T03:28:03.473915+0000 mon.a (mon.0) 1177 : cluster 0 osdmap e691: 8 total, 8 up, 8 in 2024-03-07T03:28:04.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:03 smithi189 bash[66969]: audit 2024-03-07T03:28:03.487857+0000 mon.b (mon.1) 5 : audit 1 from='client.? 172.21.15.145:0/3084881735' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:04.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:03 smithi189 bash[66969]: audit 2024-03-07T03:28:03.490265+0000 mon.a (mon.0) 1178 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:04.483 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:05.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:05 smithi145 bash[79878]: audit 2024-03-07T03:28:04.476600+0000 mon.a (mon.0) 1179 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:05.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:05 smithi145 bash[79878]: cluster 2024-03-07T03:28:04.483845+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e692: 8 total, 8 up, 8 in 2024-03-07T03:28:05.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:05 smithi145 bash[82329]: audit 2024-03-07T03:28:04.476600+0000 mon.a (mon.0) 1179 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:05.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:05 smithi145 bash[82329]: cluster 2024-03-07T03:28:04.483845+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e692: 8 total, 8 up, 8 in 2024-03-07T03:28:05.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:05 smithi189 bash[66969]: audit 2024-03-07T03:28:04.476600+0000 mon.a (mon.0) 1179 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:05.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:05 smithi189 bash[66969]: cluster 2024-03-07T03:28:04.483845+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e692: 8 total, 8 up, 8 in 2024-03-07T03:28:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:06 smithi145 bash[79878]: cluster 2024-03-07T03:28:05.019854+0000 mgr.y (mgr.34107) 487 : cluster 0 pgmap v804: 521 pgs: 15 creating+peering, 30 unknown, 476 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 388 KiB/s rd, 28 KiB/s wr, 466 op/s 2024-03-07T03:28:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:06 smithi145 bash[79878]: cluster 2024-03-07T03:28:05.498063+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e693: 8 total, 8 up, 8 in 2024-03-07T03:28:06.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:06 smithi145 bash[82329]: cluster 2024-03-07T03:28:05.019854+0000 mgr.y (mgr.34107) 487 : cluster 0 pgmap v804: 521 pgs: 15 creating+peering, 30 unknown, 476 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 388 KiB/s rd, 28 KiB/s wr, 466 op/s 2024-03-07T03:28:06.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:06 smithi145 bash[82329]: cluster 2024-03-07T03:28:05.498063+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e693: 8 total, 8 up, 8 in 2024-03-07T03:28:06.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:06 smithi189 bash[66969]: cluster 2024-03-07T03:28:05.019854+0000 mgr.y (mgr.34107) 487 : cluster 0 pgmap v804: 521 pgs: 15 creating+peering, 30 unknown, 476 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 388 KiB/s rd, 28 KiB/s wr, 466 op/s 2024-03-07T03:28:06.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:06 smithi189 bash[66969]: cluster 2024-03-07T03:28:05.498063+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e693: 8 total, 8 up, 8 in 2024-03-07T03:28:07.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:07 smithi145 bash[79878]: cluster 2024-03-07T03:28:06.504223+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e694: 8 total, 8 up, 8 in 2024-03-07T03:28:07.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:07 smithi145 bash[79878]: cluster 2024-03-07T03:28:07.471940+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e695: 8 total, 8 up, 8 in 2024-03-07T03:28:07.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:07 smithi145 bash[82329]: cluster 2024-03-07T03:28:06.504223+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e694: 8 total, 8 up, 8 in 2024-03-07T03:28:07.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:07 smithi145 bash[82329]: cluster 2024-03-07T03:28:07.471940+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e695: 8 total, 8 up, 8 in 2024-03-07T03:28:07.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:07 smithi189 bash[66969]: cluster 2024-03-07T03:28:06.504223+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e694: 8 total, 8 up, 8 in 2024-03-07T03:28:07.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:07 smithi189 bash[66969]: cluster 2024-03-07T03:28:07.471940+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e695: 8 total, 8 up, 8 in 2024-03-07T03:28:08.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:08 smithi145 bash[79878]: cluster 2024-03-07T03:28:07.021857+0000 mgr.y (mgr.34107) 488 : cluster 0 pgmap v807: 521 pgs: 9 creating+peering, 512 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 7.5 KiB/s wr, 118 op/s 2024-03-07T03:28:08.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:08 smithi145 bash[82329]: cluster 2024-03-07T03:28:07.021857+0000 mgr.y (mgr.34107) 488 : cluster 0 pgmap v807: 521 pgs: 9 creating+peering, 512 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 7.5 KiB/s wr, 118 op/s 2024-03-07T03:28:08.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:08 smithi189 bash[66969]: cluster 2024-03-07T03:28:07.021857+0000 mgr.y (mgr.34107) 488 : cluster 0 pgmap v807: 521 pgs: 9 creating+peering, 512 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 7.5 KiB/s wr, 118 op/s 2024-03-07T03:28:09.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:09 smithi145 bash[79878]: cluster 2024-03-07T03:28:08.522681+0000 mon.a (mon.0) 1184 : cluster 0 osdmap e696: 8 total, 8 up, 8 in 2024-03-07T03:28:09.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:09 smithi145 bash[82329]: cluster 2024-03-07T03:28:08.522681+0000 mon.a (mon.0) 1184 : cluster 0 osdmap e696: 8 total, 8 up, 8 in 2024-03-07T03:28:09.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:09 smithi189 bash[66969]: cluster 2024-03-07T03:28:08.522681+0000 mon.a (mon.0) 1184 : cluster 0 osdmap e696: 8 total, 8 up, 8 in 2024-03-07T03:28:10.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:10 smithi145 bash[79878]: cluster 2024-03-07T03:28:09.024178+0000 mgr.y (mgr.34107) 489 : cluster 0 pgmap v810: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 1.5 KiB/s wr, 26 op/s 2024-03-07T03:28:10.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:10 smithi145 bash[79878]: cluster 2024-03-07T03:28:09.546937+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e697: 8 total, 8 up, 8 in 2024-03-07T03:28:10.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:10 smithi145 bash[82329]: cluster 2024-03-07T03:28:09.024178+0000 mgr.y (mgr.34107) 489 : cluster 0 pgmap v810: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 1.5 KiB/s wr, 26 op/s 2024-03-07T03:28:10.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:10 smithi145 bash[82329]: cluster 2024-03-07T03:28:09.546937+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e697: 8 total, 8 up, 8 in 2024-03-07T03:28:10.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:10 smithi189 bash[66969]: cluster 2024-03-07T03:28:09.024178+0000 mgr.y (mgr.34107) 489 : cluster 0 pgmap v810: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 1.5 KiB/s wr, 26 op/s 2024-03-07T03:28:10.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:10 smithi189 bash[66969]: cluster 2024-03-07T03:28:09.546937+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e697: 8 total, 8 up, 8 in 2024-03-07T03:28:10.840 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.840 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.840 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.840 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.840 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.840 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 1 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout:child: image113 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image113/85fea21d4005 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.841 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 2 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 2 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout:child: image114 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout:child: image113 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image113/85fea21d4005 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114/85fe915d8bf8 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.842 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.843 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.843 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.843 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.843 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.843 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-28/image115 2024-03-07T03:28:10.843 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.843 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.843 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image115 2024-03-07T03:28:10.843 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image115 2024-03-07T03:28:10.843 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.843 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 3 2024-03-07T03:28:10.843 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 3 2024-03-07T03:28:10.843 INFO:tasks.workunit.client.0.smithi145.stdout:child: image114 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout:child: image113 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout:child: image115 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image113/85fea21d4005 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114/85fe915d8bf8 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image115/85fe65e1add 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.844 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 3 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 3 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout:child: image114 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout:child: image113 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout:child: image115 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image113/85fea21d4005 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114/85fe915d8bf8 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image115/85fe65e1add 2024-03-07T03:28:10.845 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-28/image116 2024-03-07T03:28:10.846 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image116 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image116 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 4 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 4 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout:child: image114 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout:child: image113 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout:child: image115 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout:child: image116 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image113/85fea21d4005 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114/85fe915d8bf8 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.847 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image115/85fe65e1add 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image116/85fe7e57c7bf 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.848 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.849 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.849 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-28/image115 2024-03-07T03:28:10.849 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:10.849 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:10.849 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image115 2024-03-07T03:28:10.849 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image115 2024-03-07T03:28:10.849 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.849 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:10.849 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-28/image116 2024-03-07T03:28:10.849 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:12.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:12 smithi189 bash[66969]: cluster 2024-03-07T03:28:11.027021+0000 mgr.y (mgr.34107) 490 : cluster 0 pgmap v812: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 4.0 KiB/s wr, 127 op/s 2024-03-07T03:28:12.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:12 smithi189 bash[66969]: cluster 2024-03-07T03:28:12.475722+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e698: 8 total, 8 up, 8 in 2024-03-07T03:28:12.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:28:12 smithi189 bash[59814]: ts=2024-03-07T03:28:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:28:12.657 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image113 2024-03-07T03:28:12.657 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image115 2024-03-07T03:28:12.657 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image116 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image116 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 4 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 4 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout:child: image114 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout:child: image113 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout:child: image115 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout:child: image116 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image113/85fea21d4005 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114/85fe915d8bf8 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image115/85fe65e1add 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image116/85fe7e57c7bf 2024-03-07T03:28:12.658 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-28/image115 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image115 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image115 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-28/image116 2024-03-07T03:28:12.659 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image115 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image116 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image116 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 3 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 3 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout:child: image114 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout:child: image115 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout:child: image116 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114/85fe915d8bf8 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image115/85fe65e1add 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image116/85fe7e57c7bf 2024-03-07T03:28:12.660 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-28/image116 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-28/image116 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image116 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 2 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 2 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout:child: image114 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout:child: image116 2024-03-07T03:28:12.661 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114/85fe915d8bf8 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-28/image116/85fe7e57c7bf 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 1 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout:child: image114 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-27/image114/85fe915d8bf8 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 0 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 0 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ListChildren (11794 ms) 2024-03-07T03:28:12.662 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ListChildrenTiered 2024-03-07T03:28:12.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:28:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:28:12] "GET /metrics HTTP/1.1" 200 65420 "" "Prometheus/2.43.0" 2024-03-07T03:28:12.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:12 smithi145 bash[79878]: cluster 2024-03-07T03:28:11.027021+0000 mgr.y (mgr.34107) 490 : cluster 0 pgmap v812: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 4.0 KiB/s wr, 127 op/s 2024-03-07T03:28:12.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:12 smithi145 bash[79878]: cluster 2024-03-07T03:28:12.475722+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e698: 8 total, 8 up, 8 in 2024-03-07T03:28:12.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:12 smithi145 bash[82329]: cluster 2024-03-07T03:28:11.027021+0000 mgr.y (mgr.34107) 490 : cluster 0 pgmap v812: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 4.0 KiB/s wr, 127 op/s 2024-03-07T03:28:12.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:12 smithi145 bash[82329]: cluster 2024-03-07T03:28:12.475722+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e698: 8 total, 8 up, 8 in 2024-03-07T03:28:14.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:14 smithi189 bash[66969]: cluster 2024-03-07T03:28:13.027993+0000 mgr.y (mgr.34107) 491 : cluster 0 pgmap v814: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 253 KiB/s rd, 7.4 KiB/s wr, 308 op/s 2024-03-07T03:28:14.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:14 smithi189 bash[66969]: cluster 2024-03-07T03:28:13.564585+0000 mon.a (mon.0) 1187 : cluster 0 osdmap e699: 8 total, 8 up, 8 in 2024-03-07T03:28:14.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:14 smithi189 bash[66969]: audit 2024-03-07T03:28:13.583175+0000 mon.a (mon.0) 1188 : audit 1 from='client.? 172.21.15.145:0/441171717' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:14.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:14 smithi189 bash[66969]: audit 2024-03-07T03:28:14.154568+0000 mon.a (mon.0) 1189 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:15.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:14 smithi145 bash[79878]: cluster 2024-03-07T03:28:13.027993+0000 mgr.y (mgr.34107) 491 : cluster 0 pgmap v814: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 253 KiB/s rd, 7.4 KiB/s wr, 308 op/s 2024-03-07T03:28:15.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:14 smithi145 bash[79878]: cluster 2024-03-07T03:28:13.564585+0000 mon.a (mon.0) 1187 : cluster 0 osdmap e699: 8 total, 8 up, 8 in 2024-03-07T03:28:15.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:14 smithi145 bash[79878]: audit 2024-03-07T03:28:13.583175+0000 mon.a (mon.0) 1188 : audit 1 from='client.? 172.21.15.145:0/441171717' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:15.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:14 smithi145 bash[79878]: audit 2024-03-07T03:28:14.154568+0000 mon.a (mon.0) 1189 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:15.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:14 smithi145 bash[82329]: cluster 2024-03-07T03:28:13.027993+0000 mgr.y (mgr.34107) 491 : cluster 0 pgmap v814: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 253 KiB/s rd, 7.4 KiB/s wr, 308 op/s 2024-03-07T03:28:15.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:14 smithi145 bash[82329]: cluster 2024-03-07T03:28:13.564585+0000 mon.a (mon.0) 1187 : cluster 0 osdmap e699: 8 total, 8 up, 8 in 2024-03-07T03:28:15.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:14 smithi145 bash[82329]: audit 2024-03-07T03:28:13.583175+0000 mon.a (mon.0) 1188 : audit 1 from='client.? 172.21.15.145:0/441171717' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:15.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:14 smithi145 bash[82329]: audit 2024-03-07T03:28:14.154568+0000 mon.a (mon.0) 1189 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:16.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:15 smithi145 bash[79878]: audit 2024-03-07T03:28:14.561201+0000 mon.a (mon.0) 1190 : audit 1 from='client.? 172.21.15.145:0/441171717' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:16.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:15 smithi145 bash[79878]: cluster 2024-03-07T03:28:14.570655+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e700: 8 total, 8 up, 8 in 2024-03-07T03:28:16.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:15 smithi145 bash[82329]: audit 2024-03-07T03:28:14.561201+0000 mon.a (mon.0) 1190 : audit 1 from='client.? 172.21.15.145:0/441171717' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:16.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:15 smithi145 bash[82329]: cluster 2024-03-07T03:28:14.570655+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e700: 8 total, 8 up, 8 in 2024-03-07T03:28:16.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:15 smithi189 bash[66969]: audit 2024-03-07T03:28:14.561201+0000 mon.a (mon.0) 1190 : audit 1 from='client.? 172.21.15.145:0/441171717' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:16.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:15 smithi189 bash[66969]: cluster 2024-03-07T03:28:14.570655+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e700: 8 total, 8 up, 8 in 2024-03-07T03:28:17.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:16 smithi145 bash[79878]: cluster 2024-03-07T03:28:15.029607+0000 mgr.y (mgr.34107) 492 : cluster 0 pgmap v817: 553 pgs: 9 creating+peering, 22 unknown, 522 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 348 KiB/s rd, 9.8 KiB/s wr, 422 op/s 2024-03-07T03:28:17.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:16 smithi145 bash[79878]: cluster 2024-03-07T03:28:15.607753+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e701: 8 total, 8 up, 8 in 2024-03-07T03:28:17.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:16 smithi145 bash[79878]: audit 2024-03-07T03:28:15.609344+0000 mon.a (mon.0) 1193 : audit 1 from='client.? 172.21.15.145:0/1989157874' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:17.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:16 smithi145 bash[79878]: audit 2024-03-07T03:28:16.086536+0000 mon.a (mon.0) 1194 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:28:17.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:16 smithi145 bash[82329]: cluster 2024-03-07T03:28:15.029607+0000 mgr.y (mgr.34107) 492 : cluster 0 pgmap v817: 553 pgs: 9 creating+peering, 22 unknown, 522 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 348 KiB/s rd, 9.8 KiB/s wr, 422 op/s 2024-03-07T03:28:17.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:16 smithi145 bash[82329]: cluster 2024-03-07T03:28:15.607753+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e701: 8 total, 8 up, 8 in 2024-03-07T03:28:17.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:16 smithi145 bash[82329]: audit 2024-03-07T03:28:15.609344+0000 mon.a (mon.0) 1193 : audit 1 from='client.? 172.21.15.145:0/1989157874' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:17.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:16 smithi145 bash[82329]: audit 2024-03-07T03:28:16.086536+0000 mon.a (mon.0) 1194 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:28:17.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:16 smithi189 bash[66969]: cluster 2024-03-07T03:28:15.029607+0000 mgr.y (mgr.34107) 492 : cluster 0 pgmap v817: 553 pgs: 9 creating+peering, 22 unknown, 522 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 348 KiB/s rd, 9.8 KiB/s wr, 422 op/s 2024-03-07T03:28:17.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:16 smithi189 bash[66969]: cluster 2024-03-07T03:28:15.607753+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e701: 8 total, 8 up, 8 in 2024-03-07T03:28:17.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:16 smithi189 bash[66969]: audit 2024-03-07T03:28:15.609344+0000 mon.a (mon.0) 1193 : audit 1 from='client.? 172.21.15.145:0/1989157874' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:17.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:16 smithi189 bash[66969]: audit 2024-03-07T03:28:16.086536+0000 mon.a (mon.0) 1194 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:28:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[82329]: audit 2024-03-07T03:28:16.603529+0000 mon.a (mon.0) 1195 : audit 1 from='client.? 172.21.15.145:0/1989157874' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[82329]: cluster 2024-03-07T03:28:16.608218+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e702: 8 total, 8 up, 8 in 2024-03-07T03:28:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[82329]: audit 2024-03-07T03:28:16.629439+0000 mon.a (mon.0) 1197 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[82329]: audit 2024-03-07T03:28:16.637888+0000 mon.a (mon.0) 1198 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[82329]: cluster 2024-03-07T03:28:17.479791+0000 mon.a (mon.0) 1199 : cluster 0 osdmap e703: 8 total, 8 up, 8 in 2024-03-07T03:28:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[82329]: audit 2024-03-07T03:28:17.482291+0000 mon.c (mon.2) 37 : audit 1 from='client.? 172.21.15.145:0/4086238571' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[82329]: audit 2024-03-07T03:28:17.483749+0000 mon.a (mon.0) 1200 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:18.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[79878]: audit 2024-03-07T03:28:16.603529+0000 mon.a (mon.0) 1195 : audit 1 from='client.? 172.21.15.145:0/1989157874' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[79878]: cluster 2024-03-07T03:28:16.608218+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e702: 8 total, 8 up, 8 in 2024-03-07T03:28:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[79878]: audit 2024-03-07T03:28:16.629439+0000 mon.a (mon.0) 1197 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[79878]: audit 2024-03-07T03:28:16.637888+0000 mon.a (mon.0) 1198 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[79878]: cluster 2024-03-07T03:28:17.479791+0000 mon.a (mon.0) 1199 : cluster 0 osdmap e703: 8 total, 8 up, 8 in 2024-03-07T03:28:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[79878]: audit 2024-03-07T03:28:17.482291+0000 mon.c (mon.2) 37 : audit 1 from='client.? 172.21.15.145:0/4086238571' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:17 smithi145 bash[79878]: audit 2024-03-07T03:28:17.483749+0000 mon.a (mon.0) 1200 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:17 smithi189 bash[66969]: audit 2024-03-07T03:28:16.603529+0000 mon.a (mon.0) 1195 : audit 1 from='client.? 172.21.15.145:0/1989157874' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:17 smithi189 bash[66969]: cluster 2024-03-07T03:28:16.608218+0000 mon.a (mon.0) 1196 : cluster 0 osdmap e702: 8 total, 8 up, 8 in 2024-03-07T03:28:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:17 smithi189 bash[66969]: audit 2024-03-07T03:28:16.629439+0000 mon.a (mon.0) 1197 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:17 smithi189 bash[66969]: audit 2024-03-07T03:28:16.637888+0000 mon.a (mon.0) 1198 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:17 smithi189 bash[66969]: cluster 2024-03-07T03:28:17.479791+0000 mon.a (mon.0) 1199 : cluster 0 osdmap e703: 8 total, 8 up, 8 in 2024-03-07T03:28:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:17 smithi189 bash[66969]: audit 2024-03-07T03:28:17.482291+0000 mon.c (mon.2) 37 : audit 1 from='client.? 172.21.15.145:0/4086238571' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:17 smithi189 bash[66969]: audit 2024-03-07T03:28:17.483749+0000 mon.a (mon.0) 1200 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:28:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:18 smithi145 bash[79878]: cluster 2024-03-07T03:28:17.031183+0000 mgr.y (mgr.34107) 493 : cluster 0 pgmap v820: 585 pgs: 20 creating+peering, 14 unknown, 551 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 238 KiB/s rd, 5.0 KiB/s wr, 292 op/s 2024-03-07T03:28:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:18 smithi145 bash[79878]: audit 2024-03-07T03:28:17.652072+0000 mon.a (mon.0) 1201 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:18 smithi145 bash[79878]: audit 2024-03-07T03:28:18.476018+0000 mon.a (mon.0) 1202 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:18 smithi145 bash[79878]: cluster 2024-03-07T03:28:18.484643+0000 mon.a (mon.0) 1203 : cluster 0 osdmap e704: 8 total, 8 up, 8 in 2024-03-07T03:28:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:18 smithi145 bash[79878]: audit 2024-03-07T03:28:18.492516+0000 mon.a (mon.0) 1204 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi145-68337-29", "tierpool":"test-librbd-smithi145-68337-31", "force_nonempty":""}]: dispatch 2024-03-07T03:28:19.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:18 smithi145 bash[82329]: cluster 2024-03-07T03:28:17.031183+0000 mgr.y (mgr.34107) 493 : cluster 0 pgmap v820: 585 pgs: 20 creating+peering, 14 unknown, 551 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 238 KiB/s rd, 5.0 KiB/s wr, 292 op/s 2024-03-07T03:28:19.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:18 smithi145 bash[82329]: audit 2024-03-07T03:28:17.652072+0000 mon.a (mon.0) 1201 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:19.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:18 smithi145 bash[82329]: audit 2024-03-07T03:28:18.476018+0000 mon.a (mon.0) 1202 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:19.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:18 smithi145 bash[82329]: cluster 2024-03-07T03:28:18.484643+0000 mon.a (mon.0) 1203 : cluster 0 osdmap e704: 8 total, 8 up, 8 in 2024-03-07T03:28:19.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:18 smithi145 bash[82329]: audit 2024-03-07T03:28:18.492516+0000 mon.a (mon.0) 1204 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi145-68337-29", "tierpool":"test-librbd-smithi145-68337-31", "force_nonempty":""}]: dispatch 2024-03-07T03:28:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:18 smithi189 bash[66969]: cluster 2024-03-07T03:28:17.031183+0000 mgr.y (mgr.34107) 493 : cluster 0 pgmap v820: 585 pgs: 20 creating+peering, 14 unknown, 551 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 238 KiB/s rd, 5.0 KiB/s wr, 292 op/s 2024-03-07T03:28:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:18 smithi189 bash[66969]: audit 2024-03-07T03:28:17.652072+0000 mon.a (mon.0) 1201 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:18 smithi189 bash[66969]: audit 2024-03-07T03:28:18.476018+0000 mon.a (mon.0) 1202 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:28:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:18 smithi189 bash[66969]: cluster 2024-03-07T03:28:18.484643+0000 mon.a (mon.0) 1203 : cluster 0 osdmap e704: 8 total, 8 up, 8 in 2024-03-07T03:28:19.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:18 smithi189 bash[66969]: audit 2024-03-07T03:28:18.492516+0000 mon.a (mon.0) 1204 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi145-68337-29", "tierpool":"test-librbd-smithi145-68337-31", "force_nonempty":""}]: dispatch 2024-03-07T03:28:20.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:20 smithi145 bash[79878]: cluster 2024-03-07T03:28:19.033825+0000 mgr.y (mgr.34107) 494 : cluster 0 pgmap v823: 617 pgs: 11 creating+peering, 21 unknown, 585 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 1.7 KiB/s wr, 141 op/s 2024-03-07T03:28:20.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:20 smithi145 bash[79878]: audit 2024-03-07T03:28:19.478981+0000 mon.a (mon.0) 1205 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi145-68337-29", "tierpool":"test-librbd-smithi145-68337-31", "force_nonempty":""}]': finished 2024-03-07T03:28:20.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:20 smithi145 bash[79878]: cluster 2024-03-07T03:28:19.484921+0000 mon.a (mon.0) 1206 : cluster 0 osdmap e705: 8 total, 8 up, 8 in 2024-03-07T03:28:20.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:20 smithi145 bash[79878]: audit 2024-03-07T03:28:19.485869+0000 mon.a (mon.0) 1207 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi145-68337-31", "mode":"writeback"}]: dispatch 2024-03-07T03:28:20.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:20 smithi145 bash[82329]: cluster 2024-03-07T03:28:19.033825+0000 mgr.y (mgr.34107) 494 : cluster 0 pgmap v823: 617 pgs: 11 creating+peering, 21 unknown, 585 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 1.7 KiB/s wr, 141 op/s 2024-03-07T03:28:20.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:20 smithi145 bash[82329]: audit 2024-03-07T03:28:19.478981+0000 mon.a (mon.0) 1205 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi145-68337-29", "tierpool":"test-librbd-smithi145-68337-31", "force_nonempty":""}]': finished 2024-03-07T03:28:20.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:20 smithi145 bash[82329]: cluster 2024-03-07T03:28:19.484921+0000 mon.a (mon.0) 1206 : cluster 0 osdmap e705: 8 total, 8 up, 8 in 2024-03-07T03:28:20.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:20 smithi145 bash[82329]: audit 2024-03-07T03:28:19.485869+0000 mon.a (mon.0) 1207 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi145-68337-31", "mode":"writeback"}]: dispatch 2024-03-07T03:28:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:20 smithi189 bash[66969]: cluster 2024-03-07T03:28:19.033825+0000 mgr.y (mgr.34107) 494 : cluster 0 pgmap v823: 617 pgs: 11 creating+peering, 21 unknown, 585 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 1.7 KiB/s wr, 141 op/s 2024-03-07T03:28:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:20 smithi189 bash[66969]: audit 2024-03-07T03:28:19.478981+0000 mon.a (mon.0) 1205 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi145-68337-29", "tierpool":"test-librbd-smithi145-68337-31", "force_nonempty":""}]': finished 2024-03-07T03:28:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:20 smithi189 bash[66969]: cluster 2024-03-07T03:28:19.484921+0000 mon.a (mon.0) 1206 : cluster 0 osdmap e705: 8 total, 8 up, 8 in 2024-03-07T03:28:20.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:20 smithi189 bash[66969]: audit 2024-03-07T03:28:19.485869+0000 mon.a (mon.0) 1207 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi145-68337-31", "mode":"writeback"}]: dispatch 2024-03-07T03:28:21.488 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:21.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:21 smithi145 bash[82329]: cluster 2024-03-07T03:28:20.479463+0000 mon.a (mon.0) 1208 : cluster 3 Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-03-07T03:28:21.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:21 smithi145 bash[82329]: audit 2024-03-07T03:28:20.481870+0000 mon.a (mon.0) 1209 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi145-68337-31", "mode":"writeback"}]': finished 2024-03-07T03:28:21.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:21 smithi145 bash[82329]: cluster 2024-03-07T03:28:20.491293+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e706: 8 total, 8 up, 8 in 2024-03-07T03:28:21.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:21 smithi145 bash[82329]: audit 2024-03-07T03:28:20.493282+0000 mon.a (mon.0) 1211 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi145-68337-29", "overlaypool":"test-librbd-smithi145-68337-31"}]: dispatch 2024-03-07T03:28:21.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:21 smithi145 bash[82329]: audit 2024-03-07T03:28:21.485162+0000 mon.a (mon.0) 1212 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi145-68337-29", "overlaypool":"test-librbd-smithi145-68337-31"}]': finished 2024-03-07T03:28:21.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:21 smithi145 bash[82329]: cluster 2024-03-07T03:28:21.497098+0000 mon.a (mon.0) 1213 : cluster 0 osdmap e707: 8 total, 8 up, 8 in 2024-03-07T03:28:21.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:21 smithi145 bash[79878]: cluster 2024-03-07T03:28:20.479463+0000 mon.a (mon.0) 1208 : cluster 3 Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-03-07T03:28:21.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:21 smithi145 bash[79878]: audit 2024-03-07T03:28:20.481870+0000 mon.a (mon.0) 1209 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi145-68337-31", "mode":"writeback"}]': finished 2024-03-07T03:28:21.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:21 smithi145 bash[79878]: cluster 2024-03-07T03:28:20.491293+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e706: 8 total, 8 up, 8 in 2024-03-07T03:28:21.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:21 smithi145 bash[79878]: audit 2024-03-07T03:28:20.493282+0000 mon.a (mon.0) 1211 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi145-68337-29", "overlaypool":"test-librbd-smithi145-68337-31"}]: dispatch 2024-03-07T03:28:21.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:21 smithi145 bash[79878]: audit 2024-03-07T03:28:21.485162+0000 mon.a (mon.0) 1212 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi145-68337-29", "overlaypool":"test-librbd-smithi145-68337-31"}]': finished 2024-03-07T03:28:21.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:21 smithi145 bash[79878]: cluster 2024-03-07T03:28:21.497098+0000 mon.a (mon.0) 1213 : cluster 0 osdmap e707: 8 total, 8 up, 8 in 2024-03-07T03:28:21.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:21 smithi189 bash[66969]: cluster 2024-03-07T03:28:20.479463+0000 mon.a (mon.0) 1208 : cluster 3 Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-03-07T03:28:21.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:21 smithi189 bash[66969]: audit 2024-03-07T03:28:20.481870+0000 mon.a (mon.0) 1209 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi145-68337-31", "mode":"writeback"}]': finished 2024-03-07T03:28:21.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:21 smithi189 bash[66969]: cluster 2024-03-07T03:28:20.491293+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e706: 8 total, 8 up, 8 in 2024-03-07T03:28:21.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:21 smithi189 bash[66969]: audit 2024-03-07T03:28:20.493282+0000 mon.a (mon.0) 1211 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi145-68337-29", "overlaypool":"test-librbd-smithi145-68337-31"}]: dispatch 2024-03-07T03:28:21.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:21 smithi189 bash[66969]: audit 2024-03-07T03:28:21.485162+0000 mon.a (mon.0) 1212 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi145-68337-29", "overlaypool":"test-librbd-smithi145-68337-31"}]': finished 2024-03-07T03:28:21.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:21 smithi189 bash[66969]: cluster 2024-03-07T03:28:21.497098+0000 mon.a (mon.0) 1213 : cluster 0 osdmap e707: 8 total, 8 up, 8 in 2024-03-07T03:28:22.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:28:22 smithi189 bash[59814]: ts=2024-03-07T03:28:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:28:22.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:22 smithi189 bash[66969]: cluster 2024-03-07T03:28:21.036165+0000 mgr.y (mgr.34107) 495 : cluster 0 pgmap v826: 617 pgs: 3 creating+peering, 4 unknown, 610 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:28:22.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:22 smithi189 bash[66969]: audit 2024-03-07T03:28:22.129898+0000 mon.a (mon.0) 1214 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:22.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:22 smithi189 bash[66969]: audit 2024-03-07T03:28:22.136955+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:22.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:22 smithi189 bash[66969]: audit 2024-03-07T03:28:22.435497+0000 mon.a (mon.0) 1216 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:22.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:22 smithi189 bash[66969]: audit 2024-03-07T03:28:22.445159+0000 mon.a (mon.0) 1217 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:22.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:22 smithi189 bash[66969]: cluster 2024-03-07T03:28:22.481935+0000 mon.a (mon.0) 1218 : cluster 0 osdmap e708: 8 total, 8 up, 8 in 2024-03-07T03:28:22.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:28:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:28:22] "GET /metrics HTTP/1.1" 200 74951 "" "Prometheus/2.43.0" 2024-03-07T03:28:22.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:22 smithi145 bash[79878]: cluster 2024-03-07T03:28:21.036165+0000 mgr.y (mgr.34107) 495 : cluster 0 pgmap v826: 617 pgs: 3 creating+peering, 4 unknown, 610 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:28:22.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:22 smithi145 bash[79878]: audit 2024-03-07T03:28:22.129898+0000 mon.a (mon.0) 1214 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:22.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:22 smithi145 bash[79878]: audit 2024-03-07T03:28:22.136955+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:22.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:22 smithi145 bash[79878]: audit 2024-03-07T03:28:22.435497+0000 mon.a (mon.0) 1216 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:22.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:22 smithi145 bash[79878]: audit 2024-03-07T03:28:22.445159+0000 mon.a (mon.0) 1217 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:22.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:22 smithi145 bash[79878]: cluster 2024-03-07T03:28:22.481935+0000 mon.a (mon.0) 1218 : cluster 0 osdmap e708: 8 total, 8 up, 8 in 2024-03-07T03:28:22.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:22 smithi145 bash[82329]: cluster 2024-03-07T03:28:21.036165+0000 mgr.y (mgr.34107) 495 : cluster 0 pgmap v826: 617 pgs: 3 creating+peering, 4 unknown, 610 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:28:22.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:22 smithi145 bash[82329]: audit 2024-03-07T03:28:22.129898+0000 mon.a (mon.0) 1214 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:22.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:22 smithi145 bash[82329]: audit 2024-03-07T03:28:22.136955+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:22.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:22 smithi145 bash[82329]: audit 2024-03-07T03:28:22.435497+0000 mon.a (mon.0) 1216 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:22.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:22 smithi145 bash[82329]: audit 2024-03-07T03:28:22.445159+0000 mon.a (mon.0) 1217 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:22.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:22 smithi145 bash[82329]: cluster 2024-03-07T03:28:22.481935+0000 mon.a (mon.0) 1218 : cluster 0 osdmap e708: 8 total, 8 up, 8 in 2024-03-07T03:28:23.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:23 smithi145 bash[79878]: audit 2024-03-07T03:28:23.073677+0000 mon.a (mon.0) 1219 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:28:23.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:23 smithi145 bash[79878]: audit 2024-03-07T03:28:23.075433+0000 mon.a (mon.0) 1220 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:28:23.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:23 smithi145 bash[79878]: audit 2024-03-07T03:28:23.085012+0000 mon.a (mon.0) 1221 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:23.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:23 smithi145 bash[79878]: cluster 2024-03-07T03:28:23.484913+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e709: 8 total, 8 up, 8 in 2024-03-07T03:28:23.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:23 smithi145 bash[82329]: audit 2024-03-07T03:28:23.073677+0000 mon.a (mon.0) 1219 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:28:23.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:23 smithi145 bash[82329]: audit 2024-03-07T03:28:23.075433+0000 mon.a (mon.0) 1220 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:28:23.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:23 smithi145 bash[82329]: audit 2024-03-07T03:28:23.085012+0000 mon.a (mon.0) 1221 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:23.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:23 smithi145 bash[82329]: cluster 2024-03-07T03:28:23.484913+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e709: 8 total, 8 up, 8 in 2024-03-07T03:28:23.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:23 smithi189 bash[66969]: audit 2024-03-07T03:28:23.073677+0000 mon.a (mon.0) 1219 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:28:23.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:23 smithi189 bash[66969]: audit 2024-03-07T03:28:23.075433+0000 mon.a (mon.0) 1220 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:28:23.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:23 smithi189 bash[66969]: audit 2024-03-07T03:28:23.085012+0000 mon.a (mon.0) 1221 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:23.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:23 smithi189 bash[66969]: cluster 2024-03-07T03:28:23.484913+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e709: 8 total, 8 up, 8 in 2024-03-07T03:28:24.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:24 smithi145 bash[82329]: cluster 2024-03-07T03:28:23.036813+0000 mgr.y (mgr.34107) 496 : cluster 0 pgmap v829: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:28:24.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:24 smithi145 bash[79878]: cluster 2024-03-07T03:28:23.036813+0000 mgr.y (mgr.34107) 496 : cluster 0 pgmap v829: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:28:24.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:24 smithi189 bash[66969]: cluster 2024-03-07T03:28:23.036813+0000 mgr.y (mgr.34107) 496 : cluster 0 pgmap v829: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:28:25.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:25 smithi145 bash[79878]: cluster 2024-03-07T03:28:24.544780+0000 mon.a (mon.0) 1223 : cluster 0 osdmap e710: 8 total, 8 up, 8 in 2024-03-07T03:28:25.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:25 smithi145 bash[82329]: cluster 2024-03-07T03:28:24.544780+0000 mon.a (mon.0) 1223 : cluster 0 osdmap e710: 8 total, 8 up, 8 in 2024-03-07T03:28:25.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:25 smithi189 bash[66969]: cluster 2024-03-07T03:28:24.544780+0000 mon.a (mon.0) 1223 : cluster 0 osdmap e710: 8 total, 8 up, 8 in 2024-03-07T03:28:26.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:26 smithi145 bash[79878]: cluster 2024-03-07T03:28:25.038515+0000 mgr.y (mgr.34107) 497 : cluster 0 pgmap v832: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 767 B/s wr, 10 op/s 2024-03-07T03:28:26.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:26 smithi145 bash[79878]: cluster 2024-03-07T03:28:25.555105+0000 mon.a (mon.0) 1224 : cluster 0 osdmap e711: 8 total, 8 up, 8 in 2024-03-07T03:28:26.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:26 smithi145 bash[82329]: cluster 2024-03-07T03:28:25.038515+0000 mgr.y (mgr.34107) 497 : cluster 0 pgmap v832: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 767 B/s wr, 10 op/s 2024-03-07T03:28:26.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:26 smithi145 bash[82329]: cluster 2024-03-07T03:28:25.555105+0000 mon.a (mon.0) 1224 : cluster 0 osdmap e711: 8 total, 8 up, 8 in 2024-03-07T03:28:26.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:26 smithi189 bash[66969]: cluster 2024-03-07T03:28:25.038515+0000 mgr.y (mgr.34107) 497 : cluster 0 pgmap v832: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 767 B/s wr, 10 op/s 2024-03-07T03:28:26.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:26 smithi189 bash[66969]: cluster 2024-03-07T03:28:25.555105+0000 mon.a (mon.0) 1224 : cluster 0 osdmap e711: 8 total, 8 up, 8 in 2024-03-07T03:28:27.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:27 smithi145 bash[82329]: cluster 2024-03-07T03:28:26.566511+0000 mon.a (mon.0) 1225 : cluster 0 osdmap e712: 8 total, 8 up, 8 in 2024-03-07T03:28:27.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:27 smithi145 bash[79878]: cluster 2024-03-07T03:28:26.566511+0000 mon.a (mon.0) 1225 : cluster 0 osdmap e712: 8 total, 8 up, 8 in 2024-03-07T03:28:27.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:27 smithi189 bash[66969]: cluster 2024-03-07T03:28:26.566511+0000 mon.a (mon.0) 1225 : cluster 0 osdmap e712: 8 total, 8 up, 8 in 2024-03-07T03:28:28.000 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.000 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.000 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.000 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.000 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.000 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 1 2024-03-07T03:28:28.000 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:28:28.000 INFO:tasks.workunit.client.0.smithi145.stdout:child: image118 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image118/85fe78010bc2 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.001 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 2 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 2 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout:child: image119 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout:child: image118 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image118/85fe78010bc2 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119/85fee3d3bd6c 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.002 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-30/image120 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image120 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image120 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 3 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 3 2024-03-07T03:28:28.003 INFO:tasks.workunit.client.0.smithi145.stdout:child: image119 2024-03-07T03:28:28.004 INFO:tasks.workunit.client.0.smithi145.stdout:child: image118 2024-03-07T03:28:28.004 INFO:tasks.workunit.client.0.smithi145.stdout:child: image120 2024-03-07T03:28:28.004 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image118/85fe78010bc2 2024-03-07T03:28:28.004 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.004 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119/85fee3d3bd6c 2024-03-07T03:28:28.004 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.004 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image120/85fec5de0e9c 2024-03-07T03:28:28.004 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.004 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.004 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.004 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.004 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.004 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 3 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 3 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout:child: image119 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout:child: image118 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout:child: image120 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image118/85fe78010bc2 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119/85fee3d3bd6c 2024-03-07T03:28:28.005 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image120/85fec5de0e9c 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.006 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-30/image121 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image121 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image121 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 4 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 4 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout:child: image119 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout:child: image118 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout:child: image120 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout:child: image121 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image118/85fe78010bc2 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.007 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119/85fee3d3bd6c 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image120/85fec5de0e9c 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image121/85fe22ad79b1 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.008 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.009 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.009 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.009 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-30/image120 2024-03-07T03:28:28.009 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.009 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:28.009 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image120 2024-03-07T03:28:28.009 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image120 2024-03-07T03:28:28.009 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.009 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:28.009 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-30/image121 2024-03-07T03:28:28.009 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:28 smithi189 bash[66969]: cluster 2024-03-07T03:28:27.041094+0000 mgr.y (mgr.34107) 498 : cluster 0 pgmap v835: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.7 KiB/s wr, 13 op/s 2024-03-07T03:28:29.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:28 smithi145 bash[79878]: cluster 2024-03-07T03:28:27.041094+0000 mgr.y (mgr.34107) 498 : cluster 0 pgmap v835: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.7 KiB/s wr, 13 op/s 2024-03-07T03:28:29.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:28 smithi145 bash[82329]: cluster 2024-03-07T03:28:27.041094+0000 mgr.y (mgr.34107) 498 : cluster 0 pgmap v835: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.7 KiB/s wr, 13 op/s 2024-03-07T03:28:30.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:30 smithi145 bash[79878]: cluster 2024-03-07T03:28:29.043985+0000 mgr.y (mgr.34107) 499 : cluster 0 pgmap v836: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 242 KiB/s rd, 6.1 KiB/s wr, 293 op/s 2024-03-07T03:28:30.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:30 smithi145 bash[79878]: audit 2024-03-07T03:28:29.219265+0000 mon.a (mon.0) 1226 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:30.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:30 smithi145 bash[79878]: cluster 2024-03-07T03:28:29.587568+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e713: 8 total, 8 up, 8 in 2024-03-07T03:28:30.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:30 smithi145 bash[79878]: audit 2024-03-07T03:28:29.826836+0000 mon.a (mon.0) 1228 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi145-68337-29"}]: dispatch 2024-03-07T03:28:30.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:30 smithi145 bash[82329]: cluster 2024-03-07T03:28:29.043985+0000 mgr.y (mgr.34107) 499 : cluster 0 pgmap v836: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 242 KiB/s rd, 6.1 KiB/s wr, 293 op/s 2024-03-07T03:28:30.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:30 smithi145 bash[82329]: audit 2024-03-07T03:28:29.219265+0000 mon.a (mon.0) 1226 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:30.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:30 smithi145 bash[82329]: cluster 2024-03-07T03:28:29.587568+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e713: 8 total, 8 up, 8 in 2024-03-07T03:28:30.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:30 smithi145 bash[82329]: audit 2024-03-07T03:28:29.826836+0000 mon.a (mon.0) 1228 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi145-68337-29"}]: dispatch 2024-03-07T03:28:30.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:30 smithi189 bash[66969]: cluster 2024-03-07T03:28:29.043985+0000 mgr.y (mgr.34107) 499 : cluster 0 pgmap v836: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 242 KiB/s rd, 6.1 KiB/s wr, 293 op/s 2024-03-07T03:28:30.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:30 smithi189 bash[66969]: audit 2024-03-07T03:28:29.219265+0000 mon.a (mon.0) 1226 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:28:30.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:30 smithi189 bash[66969]: cluster 2024-03-07T03:28:29.587568+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e713: 8 total, 8 up, 8 in 2024-03-07T03:28:30.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:30 smithi189 bash[66969]: audit 2024-03-07T03:28:29.826836+0000 mon.a (mon.0) 1228 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi145-68337-29"}]: dispatch 2024-03-07T03:28:31.581 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image118 2024-03-07T03:28:31.581 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image120 2024-03-07T03:28:31.581 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image121 2024-03-07T03:28:31.581 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image121 2024-03-07T03:28:31.581 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.581 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 4 2024-03-07T03:28:31.581 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 4 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout:child: image119 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout:child: image118 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout:child: image120 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout:child: image121 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image118/85fe78010bc2 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119/85fee3d3bd6c 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image120/85fec5de0e9c 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image121/85fe22ad79b1 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:31.582 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-30/image120 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image120 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image120 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-30/image121 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image120 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image121 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image121 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.583 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 3 2024-03-07T03:28:31.584 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 3 2024-03-07T03:28:31.584 INFO:tasks.workunit.client.0.smithi145.stdout:child: image119 2024-03-07T03:28:31.584 INFO:tasks.workunit.client.0.smithi145.stdout:child: image120 2024-03-07T03:28:31.584 INFO:tasks.workunit.client.0.smithi145.stdout:child: image121 2024-03-07T03:28:31.584 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119/85fee3d3bd6c 2024-03-07T03:28:31.584 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.584 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image120/85fec5de0e9c 2024-03-07T03:28:31.584 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.584 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image121/85fe22ad79b1 2024-03-07T03:28:31.584 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.584 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.584 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:31.584 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-30/image121 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-30/image121 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image121 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 2 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 2 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout:child: image119 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout:child: image121 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119/85fee3d3bd6c 2024-03-07T03:28:31.585 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-30/image121/85fe22ad79b1 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout:trying to find test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout:checking test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 1 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 1 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout:child: image119 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-29/image119/85fee3d3bd6c 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 0 2024-03-07T03:28:31.586 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 0 2024-03-07T03:28:31.587 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ListChildrenTiered (18924 ms) 2024-03-07T03:28:31.587 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.LockingPP 2024-03-07T03:28:31.587 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:31.752 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.LockingPP (171 ms) 2024-03-07T03:28:31.752 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.FlushAio 2024-03-07T03:28:31.752 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:32.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:31 smithi145 bash[79878]: audit 2024-03-07T03:28:30.584388+0000 mon.a (mon.0) 1229 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi145-68337-29"}]': finished 2024-03-07T03:28:32.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:31 smithi145 bash[79878]: cluster 2024-03-07T03:28:30.591611+0000 mon.a (mon.0) 1230 : cluster 0 osdmap e714: 8 total, 8 up, 8 in 2024-03-07T03:28:32.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:31 smithi145 bash[79878]: audit 2024-03-07T03:28:30.593650+0000 mon.a (mon.0) 1231 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi145-68337-29", "tierpool":"test-librbd-smithi145-68337-31"}]: dispatch 2024-03-07T03:28:32.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:31 smithi145 bash[82329]: audit 2024-03-07T03:28:30.584388+0000 mon.a (mon.0) 1229 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi145-68337-29"}]': finished 2024-03-07T03:28:32.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:31 smithi145 bash[82329]: cluster 2024-03-07T03:28:30.591611+0000 mon.a (mon.0) 1230 : cluster 0 osdmap e714: 8 total, 8 up, 8 in 2024-03-07T03:28:32.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:31 smithi145 bash[82329]: audit 2024-03-07T03:28:30.593650+0000 mon.a (mon.0) 1231 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi145-68337-29", "tierpool":"test-librbd-smithi145-68337-31"}]: dispatch 2024-03-07T03:28:32.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:31 smithi189 bash[66969]: audit 2024-03-07T03:28:30.584388+0000 mon.a (mon.0) 1229 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi145-68337-29"}]': finished 2024-03-07T03:28:32.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:31 smithi189 bash[66969]: cluster 2024-03-07T03:28:30.591611+0000 mon.a (mon.0) 1230 : cluster 0 osdmap e714: 8 total, 8 up, 8 in 2024-03-07T03:28:32.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:31 smithi189 bash[66969]: audit 2024-03-07T03:28:30.593650+0000 mon.a (mon.0) 1231 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi145-68337-29", "tierpool":"test-librbd-smithi145-68337-31"}]: dispatch 2024-03-07T03:28:32.156 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.FlushAio (405 ms) 2024-03-07T03:28:32.157 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.FlushAioPP 2024-03-07T03:28:32.157 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:32.501 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.FlushAioPP (345 ms) 2024-03-07T03:28:32.502 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ZeroLengthWrite 2024-03-07T03:28:32.502 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:32.561 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ZeroLengthWrite (59 ms) 2024-03-07T03:28:32.561 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ZeroLengthDiscard 2024-03-07T03:28:32.561 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:28:32 smithi189 bash[59814]: ts=2024-03-07T03:28:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:28:32.601 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ZeroLengthDiscard (40 ms) 2024-03-07T03:28:32.601 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ZeroLengthRead 2024-03-07T03:28:32.601 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:32.624 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ZeroLengthRead (23 ms) 2024-03-07T03:28:32.624 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.LargeCacheRead 2024-03-07T03:28:32.624 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:32.679 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.LargeCacheRead (55 ms) 2024-03-07T03:28:32.679 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestPendingAio 2024-03-07T03:28:32.679 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:33.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:28:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:28:32] "GET /metrics HTTP/1.1" 200 75035 "" "Prometheus/2.43.0" 2024-03-07T03:28:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:32 smithi145 bash[82329]: cluster 2024-03-07T03:28:31.046474+0000 mgr.y (mgr.34107) 500 : cluster 0 pgmap v839: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 381 KiB/s rd, 8.0 KiB/s wr, 455 op/s 2024-03-07T03:28:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:32 smithi145 bash[82329]: cluster 2024-03-07T03:28:31.585691+0000 mon.a (mon.0) 1232 : cluster 1 Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-03-07T03:28:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:32 smithi145 bash[82329]: audit 2024-03-07T03:28:31.593931+0000 mon.a (mon.0) 1233 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi145-68337-29", "tierpool":"test-librbd-smithi145-68337-31"}]': finished 2024-03-07T03:28:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:32 smithi145 bash[82329]: cluster 2024-03-07T03:28:31.601782+0000 mon.a (mon.0) 1234 : cluster 0 osdmap e715: 8 total, 8 up, 8 in 2024-03-07T03:28:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:32 smithi145 bash[82329]: cluster 2024-03-07T03:28:32.487338+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e716: 8 total, 8 up, 8 in 2024-03-07T03:28:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:32 smithi145 bash[79878]: cluster 2024-03-07T03:28:31.046474+0000 mgr.y (mgr.34107) 500 : cluster 0 pgmap v839: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 381 KiB/s rd, 8.0 KiB/s wr, 455 op/s 2024-03-07T03:28:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:32 smithi145 bash[79878]: cluster 2024-03-07T03:28:31.585691+0000 mon.a (mon.0) 1232 : cluster 1 Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-03-07T03:28:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:32 smithi145 bash[79878]: audit 2024-03-07T03:28:31.593931+0000 mon.a (mon.0) 1233 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi145-68337-29", "tierpool":"test-librbd-smithi145-68337-31"}]': finished 2024-03-07T03:28:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:32 smithi145 bash[79878]: cluster 2024-03-07T03:28:31.601782+0000 mon.a (mon.0) 1234 : cluster 0 osdmap e715: 8 total, 8 up, 8 in 2024-03-07T03:28:33.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:32 smithi145 bash[79878]: cluster 2024-03-07T03:28:32.487338+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e716: 8 total, 8 up, 8 in 2024-03-07T03:28:33.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:32 smithi189 bash[66969]: cluster 2024-03-07T03:28:31.046474+0000 mgr.y (mgr.34107) 500 : cluster 0 pgmap v839: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 381 KiB/s rd, 8.0 KiB/s wr, 455 op/s 2024-03-07T03:28:33.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:32 smithi189 bash[66969]: cluster 2024-03-07T03:28:31.585691+0000 mon.a (mon.0) 1232 : cluster 1 Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-03-07T03:28:33.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:32 smithi189 bash[66969]: audit 2024-03-07T03:28:31.593931+0000 mon.a (mon.0) 1233 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi145-68337-29", "tierpool":"test-librbd-smithi145-68337-31"}]': finished 2024-03-07T03:28:33.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:32 smithi189 bash[66969]: cluster 2024-03-07T03:28:31.601782+0000 mon.a (mon.0) 1234 : cluster 0 osdmap e715: 8 total, 8 up, 8 in 2024-03-07T03:28:33.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:32 smithi189 bash[66969]: cluster 2024-03-07T03:28:32.487338+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e716: 8 total, 8 up, 8 in 2024-03-07T03:28:33.156 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestPendingAio (477 ms) 2024-03-07T03:28:33.156 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyup 2024-03-07T03:28:33.156 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:33.851 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:34.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:33 smithi145 bash[79878]: audit 2024-03-07T03:28:32.651673+0000 mon.a (mon.0) 1236 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:34.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:33 smithi145 bash[82329]: audit 2024-03-07T03:28:32.651673+0000 mon.a (mon.0) 1236 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:34.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:33 smithi189 bash[66969]: audit 2024-03-07T03:28:32.651673+0000 mon.a (mon.0) 1236 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:35.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:34 smithi145 bash[79878]: cluster 2024-03-07T03:28:33.047307+0000 mgr.y (mgr.34107) 501 : cluster 0 pgmap v842: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 205 KiB/s rd, 3.7 KiB/s wr, 246 op/s 2024-03-07T03:28:35.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:34 smithi145 bash[79878]: cluster 2024-03-07T03:28:33.615512+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e717: 8 total, 8 up, 8 in 2024-03-07T03:28:35.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:34 smithi145 bash[82329]: cluster 2024-03-07T03:28:33.047307+0000 mgr.y (mgr.34107) 501 : cluster 0 pgmap v842: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 205 KiB/s rd, 3.7 KiB/s wr, 246 op/s 2024-03-07T03:28:35.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:34 smithi145 bash[82329]: cluster 2024-03-07T03:28:33.615512+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e717: 8 total, 8 up, 8 in 2024-03-07T03:28:35.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:34 smithi189 bash[66969]: cluster 2024-03-07T03:28:33.047307+0000 mgr.y (mgr.34107) 501 : cluster 0 pgmap v842: 617 pgs: 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 205 KiB/s rd, 3.7 KiB/s wr, 246 op/s 2024-03-07T03:28:35.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:34 smithi189 bash[66969]: cluster 2024-03-07T03:28:33.615512+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e717: 8 total, 8 up, 8 in 2024-03-07T03:28:35.737 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyup (2581 ms) 2024-03-07T03:28:35.737 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyupMismatch 2024-03-07T03:28:35.737 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:36.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:35 smithi145 bash[79878]: cluster 2024-03-07T03:28:34.623192+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e718: 8 total, 8 up, 8 in 2024-03-07T03:28:36.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:35 smithi145 bash[82329]: cluster 2024-03-07T03:28:34.623192+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e718: 8 total, 8 up, 8 in 2024-03-07T03:28:36.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:35 smithi189 bash[66969]: cluster 2024-03-07T03:28:34.623192+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e718: 8 total, 8 up, 8 in 2024-03-07T03:28:36.924 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:36 smithi145 bash[79878]: cluster 2024-03-07T03:28:35.049051+0000 mgr.y (mgr.34107) 502 : cluster 0 pgmap v845: 617 pgs: 617 active+clean; 303 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 210 KiB/s rd, 145 KiB/s wr, 408 op/s 2024-03-07T03:28:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:36 smithi145 bash[79878]: cluster 2024-03-07T03:28:35.636078+0000 mon.a (mon.0) 1239 : cluster 0 osdmap e719: 8 total, 8 up, 8 in 2024-03-07T03:28:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:36 smithi145 bash[82329]: cluster 2024-03-07T03:28:35.049051+0000 mgr.y (mgr.34107) 502 : cluster 0 pgmap v845: 617 pgs: 617 active+clean; 303 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 210 KiB/s rd, 145 KiB/s wr, 408 op/s 2024-03-07T03:28:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:36 smithi145 bash[82329]: cluster 2024-03-07T03:28:35.636078+0000 mon.a (mon.0) 1239 : cluster 0 osdmap e719: 8 total, 8 up, 8 in 2024-03-07T03:28:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:36 smithi189 bash[66969]: cluster 2024-03-07T03:28:35.049051+0000 mgr.y (mgr.34107) 502 : cluster 0 pgmap v845: 617 pgs: 617 active+clean; 303 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 210 KiB/s rd, 145 KiB/s wr, 408 op/s 2024-03-07T03:28:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:36 smithi189 bash[66969]: cluster 2024-03-07T03:28:35.636078+0000 mon.a (mon.0) 1239 : cluster 0 osdmap e719: 8 total, 8 up, 8 in 2024-03-07T03:28:38.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:37 smithi145 bash[79878]: cluster 2024-03-07T03:28:36.636445+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e720: 8 total, 8 up, 8 in 2024-03-07T03:28:38.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:37 smithi145 bash[79878]: cluster 2024-03-07T03:28:37.487829+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e721: 8 total, 8 up, 8 in 2024-03-07T03:28:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:37 smithi145 bash[82329]: cluster 2024-03-07T03:28:36.636445+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e720: 8 total, 8 up, 8 in 2024-03-07T03:28:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:37 smithi145 bash[82329]: cluster 2024-03-07T03:28:37.487829+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e721: 8 total, 8 up, 8 in 2024-03-07T03:28:38.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:37 smithi189 bash[66969]: cluster 2024-03-07T03:28:36.636445+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e720: 8 total, 8 up, 8 in 2024-03-07T03:28:38.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:37 smithi189 bash[66969]: cluster 2024-03-07T03:28:37.487829+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e721: 8 total, 8 up, 8 in 2024-03-07T03:28:38.774 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyupMismatch (3037 ms) 2024-03-07T03:28:38.774 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.Flatten 2024-03-07T03:28:38.774 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:38 smithi145 bash[79878]: cluster 2024-03-07T03:28:37.050946+0000 mgr.y (mgr.34107) 503 : cluster 0 pgmap v848: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 826 KiB/s rd, 730 KiB/s wr, 610 op/s 2024-03-07T03:28:39.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:38 smithi145 bash[82329]: cluster 2024-03-07T03:28:37.050946+0000 mgr.y (mgr.34107) 503 : cluster 0 pgmap v848: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 826 KiB/s rd, 730 KiB/s wr, 610 op/s 2024-03-07T03:28:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:38 smithi189 bash[66969]: cluster 2024-03-07T03:28:37.050946+0000 mgr.y (mgr.34107) 503 : cluster 0 pgmap v848: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 826 KiB/s rd, 730 KiB/s wr, 610 op/s 2024-03-07T03:28:39.909 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.Flatten (1135 ms) 2024-03-07T03:28:39.910 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.Sparsify 2024-03-07T03:28:39.910 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:39.997 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 32768 2024-03-07T03:28:39.997 INFO:tasks.workunit.client.0.smithi145.stdout:read: 32768 2024-03-07T03:28:39.997 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.Sparsify (87 ms) 2024-03-07T03:28:39.997 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.SparsifyPP 2024-03-07T03:28:39.997 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:40.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:39 smithi145 bash[79878]: cluster 2024-03-07T03:28:38.669788+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e722: 8 total, 8 up, 8 in 2024-03-07T03:28:40.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:39 smithi145 bash[82329]: cluster 2024-03-07T03:28:38.669788+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e722: 8 total, 8 up, 8 in 2024-03-07T03:28:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:39 smithi189 bash[66969]: cluster 2024-03-07T03:28:38.669788+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e722: 8 total, 8 up, 8 in 2024-03-07T03:28:40.087 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.SparsifyPP (91 ms) 2024-03-07T03:28:40.087 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.SnapshotLimit 2024-03-07T03:28:40.088 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:41.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:40 smithi145 bash[79878]: cluster 2024-03-07T03:28:39.053708+0000 mgr.y (mgr.34107) 504 : cluster 0 pgmap v851: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 751 KiB/s rd, 598 KiB/s wr, 384 op/s 2024-03-07T03:28:41.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:40 smithi145 bash[79878]: cluster 2024-03-07T03:28:39.680272+0000 mon.a (mon.0) 1243 : cluster 0 osdmap e723: 8 total, 8 up, 8 in 2024-03-07T03:28:41.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:40 smithi145 bash[82329]: cluster 2024-03-07T03:28:39.053708+0000 mgr.y (mgr.34107) 504 : cluster 0 pgmap v851: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 751 KiB/s rd, 598 KiB/s wr, 384 op/s 2024-03-07T03:28:41.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:40 smithi145 bash[82329]: cluster 2024-03-07T03:28:39.680272+0000 mon.a (mon.0) 1243 : cluster 0 osdmap e723: 8 total, 8 up, 8 in 2024-03-07T03:28:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:40 smithi189 bash[66969]: cluster 2024-03-07T03:28:39.053708+0000 mgr.y (mgr.34107) 504 : cluster 0 pgmap v851: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 751 KiB/s rd, 598 KiB/s wr, 384 op/s 2024-03-07T03:28:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:40 smithi189 bash[66969]: cluster 2024-03-07T03:28:39.680272+0000 mon.a (mon.0) 1243 : cluster 0 osdmap e723: 8 total, 8 up, 8 in 2024-03-07T03:28:42.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:41 smithi145 bash[79878]: cluster 2024-03-07T03:28:40.693925+0000 mon.a (mon.0) 1244 : cluster 0 osdmap e724: 8 total, 8 up, 8 in 2024-03-07T03:28:42.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:41 smithi145 bash[82329]: cluster 2024-03-07T03:28:40.693925+0000 mon.a (mon.0) 1244 : cluster 0 osdmap e724: 8 total, 8 up, 8 in 2024-03-07T03:28:42.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:41 smithi189 bash[66969]: cluster 2024-03-07T03:28:40.693925+0000 mon.a (mon.0) 1244 : cluster 0 osdmap e724: 8 total, 8 up, 8 in 2024-03-07T03:28:42.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:28:42 smithi189 bash[59814]: ts=2024-03-07T03:28:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:28:43.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:28:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:28:42] "GET /metrics HTTP/1.1" 200 75035 "" "Prometheus/2.43.0" 2024-03-07T03:28:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:42 smithi145 bash[79878]: cluster 2024-03-07T03:28:41.055981+0000 mgr.y (mgr.34107) 505 : cluster 0 pgmap v854: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 238 KiB/s rd, 33 KiB/s wr, 292 op/s 2024-03-07T03:28:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:42 smithi145 bash[79878]: cluster 2024-03-07T03:28:41.705744+0000 mon.a (mon.0) 1245 : cluster 0 osdmap e725: 8 total, 8 up, 8 in 2024-03-07T03:28:43.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:42 smithi145 bash[82329]: cluster 2024-03-07T03:28:41.055981+0000 mgr.y (mgr.34107) 505 : cluster 0 pgmap v854: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 238 KiB/s rd, 33 KiB/s wr, 292 op/s 2024-03-07T03:28:43.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:42 smithi145 bash[82329]: cluster 2024-03-07T03:28:41.705744+0000 mon.a (mon.0) 1245 : cluster 0 osdmap e725: 8 total, 8 up, 8 in 2024-03-07T03:28:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:42 smithi189 bash[66969]: cluster 2024-03-07T03:28:41.055981+0000 mgr.y (mgr.34107) 505 : cluster 0 pgmap v854: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 238 KiB/s rd, 33 KiB/s wr, 292 op/s 2024-03-07T03:28:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:42 smithi189 bash[66969]: cluster 2024-03-07T03:28:41.705744+0000 mon.a (mon.0) 1245 : cluster 0 osdmap e725: 8 total, 8 up, 8 in 2024-03-07T03:28:44.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:43 smithi145 bash[79878]: cluster 2024-03-07T03:28:42.716089+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e726: 8 total, 8 up, 8 in 2024-03-07T03:28:44.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:43 smithi145 bash[82329]: cluster 2024-03-07T03:28:42.716089+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e726: 8 total, 8 up, 8 in 2024-03-07T03:28:44.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:43 smithi189 bash[66969]: cluster 2024-03-07T03:28:42.716089+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e726: 8 total, 8 up, 8 in 2024-03-07T03:28:44.798 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.SnapshotLimit (4710 ms) 2024-03-07T03:28:44.798 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.SnapshotLimitPP 2024-03-07T03:28:44.798 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:45.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:44 smithi145 bash[79878]: cluster 2024-03-07T03:28:43.057083+0000 mgr.y (mgr.34107) 506 : cluster 0 pgmap v857: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 24 KiB/s wr, 152 op/s 2024-03-07T03:28:45.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:44 smithi145 bash[79878]: cluster 2024-03-07T03:28:43.724700+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e727: 8 total, 8 up, 8 in 2024-03-07T03:28:45.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:44 smithi145 bash[82329]: cluster 2024-03-07T03:28:43.057083+0000 mgr.y (mgr.34107) 506 : cluster 0 pgmap v857: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 24 KiB/s wr, 152 op/s 2024-03-07T03:28:45.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:44 smithi145 bash[82329]: cluster 2024-03-07T03:28:43.724700+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e727: 8 total, 8 up, 8 in 2024-03-07T03:28:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:44 smithi189 bash[66969]: cluster 2024-03-07T03:28:43.057083+0000 mgr.y (mgr.34107) 506 : cluster 0 pgmap v857: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 24 KiB/s wr, 152 op/s 2024-03-07T03:28:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:44 smithi189 bash[66969]: cluster 2024-03-07T03:28:43.724700+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e727: 8 total, 8 up, 8 in 2024-03-07T03:28:46.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:45 smithi145 bash[79878]: cluster 2024-03-07T03:28:44.727323+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e728: 8 total, 8 up, 8 in 2024-03-07T03:28:46.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:45 smithi145 bash[82329]: cluster 2024-03-07T03:28:44.727323+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e728: 8 total, 8 up, 8 in 2024-03-07T03:28:46.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:45 smithi189 bash[66969]: cluster 2024-03-07T03:28:44.727323+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e728: 8 total, 8 up, 8 in 2024-03-07T03:28:47.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:46 smithi145 bash[79878]: cluster 2024-03-07T03:28:45.058927+0000 mgr.y (mgr.34107) 507 : cluster 0 pgmap v860: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 19 KiB/s wr, 123 op/s 2024-03-07T03:28:47.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:46 smithi145 bash[79878]: cluster 2024-03-07T03:28:45.749336+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e729: 8 total, 8 up, 8 in 2024-03-07T03:28:47.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:46 smithi145 bash[82329]: cluster 2024-03-07T03:28:45.058927+0000 mgr.y (mgr.34107) 507 : cluster 0 pgmap v860: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 19 KiB/s wr, 123 op/s 2024-03-07T03:28:47.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:46 smithi145 bash[82329]: cluster 2024-03-07T03:28:45.749336+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e729: 8 total, 8 up, 8 in 2024-03-07T03:28:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:46 smithi189 bash[66969]: cluster 2024-03-07T03:28:45.058927+0000 mgr.y (mgr.34107) 507 : cluster 0 pgmap v860: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 19 KiB/s wr, 123 op/s 2024-03-07T03:28:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:46 smithi189 bash[66969]: cluster 2024-03-07T03:28:45.749336+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e729: 8 total, 8 up, 8 in 2024-03-07T03:28:48.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:47 smithi145 bash[79878]: cluster 2024-03-07T03:28:46.751719+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e730: 8 total, 8 up, 8 in 2024-03-07T03:28:48.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:47 smithi145 bash[79878]: audit 2024-03-07T03:28:47.652629+0000 mon.a (mon.0) 1251 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:48.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:47 smithi145 bash[82329]: cluster 2024-03-07T03:28:46.751719+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e730: 8 total, 8 up, 8 in 2024-03-07T03:28:48.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:47 smithi145 bash[82329]: audit 2024-03-07T03:28:47.652629+0000 mon.a (mon.0) 1251 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:47 smithi189 bash[66969]: cluster 2024-03-07T03:28:46.751719+0000 mon.a (mon.0) 1250 : cluster 0 osdmap e730: 8 total, 8 up, 8 in 2024-03-07T03:28:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:47 smithi189 bash[66969]: audit 2024-03-07T03:28:47.652629+0000 mon.a (mon.0) 1251 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:48 smithi145 bash[79878]: cluster 2024-03-07T03:28:47.060730+0000 mgr.y (mgr.34107) 508 : cluster 0 pgmap v863: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 90 KiB/s rd, 16 KiB/s wr, 120 op/s 2024-03-07T03:28:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:48 smithi145 bash[79878]: cluster 2024-03-07T03:28:47.778709+0000 mon.a (mon.0) 1252 : cluster 0 osdmap e731: 8 total, 8 up, 8 in 2024-03-07T03:28:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:48 smithi145 bash[82329]: cluster 2024-03-07T03:28:47.060730+0000 mgr.y (mgr.34107) 508 : cluster 0 pgmap v863: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 90 KiB/s rd, 16 KiB/s wr, 120 op/s 2024-03-07T03:28:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:48 smithi145 bash[82329]: cluster 2024-03-07T03:28:47.778709+0000 mon.a (mon.0) 1252 : cluster 0 osdmap e731: 8 total, 8 up, 8 in 2024-03-07T03:28:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:48 smithi189 bash[66969]: cluster 2024-03-07T03:28:47.060730+0000 mgr.y (mgr.34107) 508 : cluster 0 pgmap v863: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 90 KiB/s rd, 16 KiB/s wr, 120 op/s 2024-03-07T03:28:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:48 smithi189 bash[66969]: cluster 2024-03-07T03:28:47.778709+0000 mon.a (mon.0) 1252 : cluster 0 osdmap e731: 8 total, 8 up, 8 in 2024-03-07T03:28:49.856 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.SnapshotLimitPP (5058 ms) 2024-03-07T03:28:49.856 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.RebuildObjectMapViaLockOwner 2024-03-07T03:28:49.856 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:49.993 INFO:tasks.workunit.client.0.smithi145.stdout:0.00% done 2024-03-07T03:28:49.993 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.RebuildObjectMapViaLockOwner (137 ms) 2024-03-07T03:28:49.993 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.RenameViaLockOwner 2024-03-07T03:28:49.993 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:50.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:49 smithi145 bash[79878]: cluster 2024-03-07T03:28:48.780398+0000 mon.a (mon.0) 1253 : cluster 0 osdmap e732: 8 total, 8 up, 8 in 2024-03-07T03:28:50.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:49 smithi145 bash[82329]: cluster 2024-03-07T03:28:48.780398+0000 mon.a (mon.0) 1253 : cluster 0 osdmap e732: 8 total, 8 up, 8 in 2024-03-07T03:28:50.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:49 smithi189 bash[66969]: cluster 2024-03-07T03:28:48.780398+0000 mon.a (mon.0) 1253 : cluster 0 osdmap e732: 8 total, 8 up, 8 in 2024-03-07T03:28:50.156 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.RenameViaLockOwner (163 ms) 2024-03-07T03:28:50.156 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.SnapCreateViaLockOwner 2024-03-07T03:28:50.156 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:50.885 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.SnapCreateViaLockOwner (729 ms) 2024-03-07T03:28:50.886 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.SnapRemoveViaLockOwner 2024-03-07T03:28:50.886 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:51.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:50 smithi145 bash[79878]: cluster 2024-03-07T03:28:49.062472+0000 mgr.y (mgr.34107) 509 : cluster 0 pgmap v866: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 2.2 KiB/s wr, 25 op/s 2024-03-07T03:28:51.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:50 smithi145 bash[79878]: cluster 2024-03-07T03:28:49.800214+0000 mon.a (mon.0) 1254 : cluster 0 osdmap e733: 8 total, 8 up, 8 in 2024-03-07T03:28:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:50 smithi145 bash[82329]: cluster 2024-03-07T03:28:49.062472+0000 mgr.y (mgr.34107) 509 : cluster 0 pgmap v866: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 2.2 KiB/s wr, 25 op/s 2024-03-07T03:28:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:50 smithi145 bash[82329]: cluster 2024-03-07T03:28:49.800214+0000 mon.a (mon.0) 1254 : cluster 0 osdmap e733: 8 total, 8 up, 8 in 2024-03-07T03:28:51.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:50 smithi189 bash[66969]: cluster 2024-03-07T03:28:49.062472+0000 mgr.y (mgr.34107) 509 : cluster 0 pgmap v866: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 2.2 KiB/s wr, 25 op/s 2024-03-07T03:28:51.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:50 smithi189 bash[66969]: cluster 2024-03-07T03:28:49.800214+0000 mon.a (mon.0) 1254 : cluster 0 osdmap e733: 8 total, 8 up, 8 in 2024-03-07T03:28:52.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:51 smithi145 bash[79878]: cluster 2024-03-07T03:28:50.812125+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e734: 8 total, 8 up, 8 in 2024-03-07T03:28:52.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:51 smithi145 bash[82329]: cluster 2024-03-07T03:28:50.812125+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e734: 8 total, 8 up, 8 in 2024-03-07T03:28:52.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:51 smithi189 bash[66969]: cluster 2024-03-07T03:28:50.812125+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e734: 8 total, 8 up, 8 in 2024-03-07T03:28:52.546 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.SnapRemoveViaLockOwner (1661 ms) 2024-03-07T03:28:52.547 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.UpdateFeaturesViaLockOwner 2024-03-07T03:28:52.547 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:28:52 smithi189 bash[59814]: ts=2024-03-07T03:28:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:28:52.674 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.UpdateFeaturesViaLockOwner (128 ms) 2024-03-07T03:28:52.674 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.EnableJournalingViaLockOwner 2024-03-07T03:28:52.674 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_librbd.cc:9061: Skipped 2024-03-07T03:28:52.674 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:28:52.674 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestLibRBD.EnableJournalingViaLockOwner (0 ms) 2024-03-07T03:28:52.674 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.SnapRemove2 2024-03-07T03:28:52.674 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:53.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:28:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:28:52] "GET /metrics HTTP/1.1" 200 75033 "" "Prometheus/2.43.0" 2024-03-07T03:28:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:52 smithi145 bash[79878]: cluster 2024-03-07T03:28:51.065326+0000 mgr.y (mgr.34107) 510 : cluster 0 pgmap v869: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 1.5 KiB/s wr, 41 op/s 2024-03-07T03:28:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:52 smithi145 bash[79878]: cluster 2024-03-07T03:28:51.823578+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e735: 8 total, 8 up, 8 in 2024-03-07T03:28:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:52 smithi145 bash[79878]: cluster 2024-03-07T03:28:52.492621+0000 mon.a (mon.0) 1257 : cluster 0 osdmap e736: 8 total, 8 up, 8 in 2024-03-07T03:28:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:52 smithi145 bash[82329]: cluster 2024-03-07T03:28:51.065326+0000 mgr.y (mgr.34107) 510 : cluster 0 pgmap v869: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 1.5 KiB/s wr, 41 op/s 2024-03-07T03:28:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:52 smithi145 bash[82329]: cluster 2024-03-07T03:28:51.823578+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e735: 8 total, 8 up, 8 in 2024-03-07T03:28:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:52 smithi145 bash[82329]: cluster 2024-03-07T03:28:52.492621+0000 mon.a (mon.0) 1257 : cluster 0 osdmap e736: 8 total, 8 up, 8 in 2024-03-07T03:28:53.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:52 smithi189 bash[66969]: cluster 2024-03-07T03:28:51.065326+0000 mgr.y (mgr.34107) 510 : cluster 0 pgmap v869: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 1.5 KiB/s wr, 41 op/s 2024-03-07T03:28:53.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:52 smithi189 bash[66969]: cluster 2024-03-07T03:28:51.823578+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e735: 8 total, 8 up, 8 in 2024-03-07T03:28:53.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:52 smithi189 bash[66969]: cluster 2024-03-07T03:28:52.492621+0000 mon.a (mon.0) 1257 : cluster 0 osdmap e736: 8 total, 8 up, 8 in 2024-03-07T03:28:54.555 INFO:tasks.workunit.client.0.smithi145.stdout:100.00% done 2024-03-07T03:28:54.556 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.SnapRemove2 (1881 ms) 2024-03-07T03:28:54.556 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.SnapRenameViaLockOwner 2024-03-07T03:28:54.556 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:54.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:54 smithi145 bash[79878]: cluster 2024-03-07T03:28:53.066516+0000 mgr.y (mgr.34107) 511 : cluster 0 pgmap v872: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 3.5 KiB/s wr, 52 op/s 2024-03-07T03:28:54.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:54 smithi145 bash[79878]: cluster 2024-03-07T03:28:53.495205+0000 mon.a (mon.0) 1258 : cluster 0 osdmap e737: 8 total, 8 up, 8 in 2024-03-07T03:28:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:54 smithi145 bash[82329]: cluster 2024-03-07T03:28:53.066516+0000 mgr.y (mgr.34107) 511 : cluster 0 pgmap v872: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 3.5 KiB/s wr, 52 op/s 2024-03-07T03:28:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:54 smithi145 bash[82329]: cluster 2024-03-07T03:28:53.495205+0000 mon.a (mon.0) 1258 : cluster 0 osdmap e737: 8 total, 8 up, 8 in 2024-03-07T03:28:54.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:54 smithi189 bash[66969]: cluster 2024-03-07T03:28:53.066516+0000 mgr.y (mgr.34107) 511 : cluster 0 pgmap v872: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 3.5 KiB/s wr, 52 op/s 2024-03-07T03:28:54.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:54 smithi189 bash[66969]: cluster 2024-03-07T03:28:53.495205+0000 mon.a (mon.0) 1258 : cluster 0 osdmap e737: 8 total, 8 up, 8 in 2024-03-07T03:28:55.627 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.SnapRenameViaLockOwner (1072 ms) 2024-03-07T03:28:55.627 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.SnapProtectViaLockOwner 2024-03-07T03:28:55.627 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:55.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:55 smithi145 bash[79878]: cluster 2024-03-07T03:28:54.505831+0000 mon.a (mon.0) 1259 : cluster 0 osdmap e738: 8 total, 8 up, 8 in 2024-03-07T03:28:55.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:55 smithi145 bash[82329]: cluster 2024-03-07T03:28:54.505831+0000 mon.a (mon.0) 1259 : cluster 0 osdmap e738: 8 total, 8 up, 8 in 2024-03-07T03:28:55.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:55 smithi189 bash[66969]: cluster 2024-03-07T03:28:54.505831+0000 mon.a (mon.0) 1259 : cluster 0 osdmap e738: 8 total, 8 up, 8 in 2024-03-07T03:28:56.605 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.SnapProtectViaLockOwner (978 ms) 2024-03-07T03:28:56.605 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.SnapUnprotectViaLockOwner 2024-03-07T03:28:56.605 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:56.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:56 smithi145 bash[79878]: cluster 2024-03-07T03:28:55.068232+0000 mgr.y (mgr.34107) 512 : cluster 0 pgmap v875: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 12 KiB/s wr, 158 op/s 2024-03-07T03:28:56.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:56 smithi145 bash[79878]: cluster 2024-03-07T03:28:55.516391+0000 mon.a (mon.0) 1260 : cluster 0 osdmap e739: 8 total, 8 up, 8 in 2024-03-07T03:28:56.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:56 smithi145 bash[82329]: cluster 2024-03-07T03:28:55.068232+0000 mgr.y (mgr.34107) 512 : cluster 0 pgmap v875: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 12 KiB/s wr, 158 op/s 2024-03-07T03:28:56.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:56 smithi145 bash[82329]: cluster 2024-03-07T03:28:55.516391+0000 mon.a (mon.0) 1260 : cluster 0 osdmap e739: 8 total, 8 up, 8 in 2024-03-07T03:28:56.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:56 smithi189 bash[66969]: cluster 2024-03-07T03:28:55.068232+0000 mgr.y (mgr.34107) 512 : cluster 0 pgmap v875: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 12 KiB/s wr, 158 op/s 2024-03-07T03:28:56.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:56 smithi189 bash[66969]: cluster 2024-03-07T03:28:55.516391+0000 mon.a (mon.0) 1260 : cluster 0 osdmap e739: 8 total, 8 up, 8 in 2024-03-07T03:28:57.662 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.SnapUnprotectViaLockOwner (1057 ms) 2024-03-07T03:28:57.662 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.FlattenViaLockOwner 2024-03-07T03:28:57.662 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:57.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:57 smithi145 bash[79878]: cluster 2024-03-07T03:28:56.522686+0000 mon.a (mon.0) 1261 : cluster 0 osdmap e740: 8 total, 8 up, 8 in 2024-03-07T03:28:57.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:57 smithi145 bash[79878]: cluster 2024-03-07T03:28:57.498232+0000 mon.a (mon.0) 1262 : cluster 0 osdmap e741: 8 total, 8 up, 8 in 2024-03-07T03:28:57.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:57 smithi145 bash[82329]: cluster 2024-03-07T03:28:56.522686+0000 mon.a (mon.0) 1261 : cluster 0 osdmap e740: 8 total, 8 up, 8 in 2024-03-07T03:28:57.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:57 smithi145 bash[82329]: cluster 2024-03-07T03:28:57.498232+0000 mon.a (mon.0) 1262 : cluster 0 osdmap e741: 8 total, 8 up, 8 in 2024-03-07T03:28:57.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:57 smithi189 bash[66969]: cluster 2024-03-07T03:28:56.522686+0000 mon.a (mon.0) 1261 : cluster 0 osdmap e740: 8 total, 8 up, 8 in 2024-03-07T03:28:57.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:57 smithi189 bash[66969]: cluster 2024-03-07T03:28:57.498232+0000 mon.a (mon.0) 1262 : cluster 0 osdmap e741: 8 total, 8 up, 8 in 2024-03-07T03:28:58.800 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.FlattenViaLockOwner (1138 ms) 2024-03-07T03:28:58.800 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ResizeViaLockOwner 2024-03-07T03:28:58.801 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:58.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:58 smithi145 bash[79878]: cluster 2024-03-07T03:28:57.070613+0000 mgr.y (mgr.34107) 513 : cluster 0 pgmap v878: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 194 KiB/s rd, 16 KiB/s wr, 265 op/s 2024-03-07T03:28:58.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:58 smithi145 bash[82329]: cluster 2024-03-07T03:28:57.070613+0000 mgr.y (mgr.34107) 513 : cluster 0 pgmap v878: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 194 KiB/s rd, 16 KiB/s wr, 265 op/s 2024-03-07T03:28:58.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:58 smithi189 bash[66969]: cluster 2024-03-07T03:28:57.070613+0000 mgr.y (mgr.34107) 513 : cluster 0 pgmap v878: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 194 KiB/s rd, 16 KiB/s wr, 265 op/s 2024-03-07T03:28:58.908 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ResizeViaLockOwner (108 ms) 2024-03-07T03:28:58.908 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.SparsifyViaLockOwner 2024-03-07T03:28:58.908 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:59.006 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.SparsifyViaLockOwner (98 ms) 2024-03-07T03:28:59.006 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ObjectMapConsistentSnap 2024-03-07T03:28:59.006 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:28:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:28:59 smithi189 bash[66969]: cluster 2024-03-07T03:28:58.549084+0000 mon.a (mon.0) 1263 : cluster 0 osdmap e742: 8 total, 8 up, 8 in 2024-03-07T03:29:00.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:28:59 smithi145 bash[79878]: cluster 2024-03-07T03:28:58.549084+0000 mon.a (mon.0) 1263 : cluster 0 osdmap e742: 8 total, 8 up, 8 in 2024-03-07T03:29:00.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:28:59 smithi145 bash[82329]: cluster 2024-03-07T03:28:58.549084+0000 mon.a (mon.0) 1263 : cluster 0 osdmap e742: 8 total, 8 up, 8 in 2024-03-07T03:29:01.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:00 smithi145 bash[79878]: cluster 2024-03-07T03:28:59.073419+0000 mgr.y (mgr.34107) 514 : cluster 0 pgmap v881: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 202 KiB/s rd, 13 KiB/s wr, 267 op/s 2024-03-07T03:29:01.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:00 smithi145 bash[79878]: cluster 2024-03-07T03:28:59.578999+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e743: 8 total, 8 up, 8 in 2024-03-07T03:29:01.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:00 smithi145 bash[82329]: cluster 2024-03-07T03:28:59.073419+0000 mgr.y (mgr.34107) 514 : cluster 0 pgmap v881: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 202 KiB/s rd, 13 KiB/s wr, 267 op/s 2024-03-07T03:29:01.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:00 smithi145 bash[82329]: cluster 2024-03-07T03:28:59.578999+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e743: 8 total, 8 up, 8 in 2024-03-07T03:29:01.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:00 smithi189 bash[66969]: cluster 2024-03-07T03:28:59.073419+0000 mgr.y (mgr.34107) 514 : cluster 0 pgmap v881: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 202 KiB/s rd, 13 KiB/s wr, 267 op/s 2024-03-07T03:29:01.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:00 smithi189 bash[66969]: cluster 2024-03-07T03:28:59.578999+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e743: 8 total, 8 up, 8 in 2024-03-07T03:29:01.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:01 smithi189 bash[66969]: cluster 2024-03-07T03:29:00.574615+0000 mon.a (mon.0) 1265 : cluster 0 osdmap e744: 8 total, 8 up, 8 in 2024-03-07T03:29:02.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:01 smithi145 bash[79878]: cluster 2024-03-07T03:29:00.574615+0000 mon.a (mon.0) 1265 : cluster 0 osdmap e744: 8 total, 8 up, 8 in 2024-03-07T03:29:02.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:01 smithi145 bash[82329]: cluster 2024-03-07T03:29:00.574615+0000 mon.a (mon.0) 1265 : cluster 0 osdmap e744: 8 total, 8 up, 8 in 2024-03-07T03:29:02.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:29:02 smithi189 bash[59814]: ts=2024-03-07T03:29:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:29:03.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:29:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:29:02] "GET /metrics HTTP/1.1" 200 75033 "" "Prometheus/2.43.0" 2024-03-07T03:29:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:02 smithi145 bash[79878]: cluster 2024-03-07T03:29:01.075755+0000 mgr.y (mgr.34107) 515 : cluster 0 pgmap v884: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 8.7 KiB/s wr, 211 op/s 2024-03-07T03:29:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:02 smithi145 bash[79878]: cluster 2024-03-07T03:29:01.597803+0000 mon.a (mon.0) 1266 : cluster 0 osdmap e745: 8 total, 8 up, 8 in 2024-03-07T03:29:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:02 smithi145 bash[79878]: cluster 2024-03-07T03:29:02.500031+0000 mon.a (mon.0) 1267 : cluster 0 osdmap e746: 8 total, 8 up, 8 in 2024-03-07T03:29:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:02 smithi145 bash[82329]: cluster 2024-03-07T03:29:01.075755+0000 mgr.y (mgr.34107) 515 : cluster 0 pgmap v884: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 8.7 KiB/s wr, 211 op/s 2024-03-07T03:29:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:02 smithi145 bash[82329]: cluster 2024-03-07T03:29:01.597803+0000 mon.a (mon.0) 1266 : cluster 0 osdmap e745: 8 total, 8 up, 8 in 2024-03-07T03:29:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:02 smithi145 bash[82329]: cluster 2024-03-07T03:29:02.500031+0000 mon.a (mon.0) 1267 : cluster 0 osdmap e746: 8 total, 8 up, 8 in 2024-03-07T03:29:03.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:02 smithi189 bash[66969]: cluster 2024-03-07T03:29:01.075755+0000 mgr.y (mgr.34107) 515 : cluster 0 pgmap v884: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 8.7 KiB/s wr, 211 op/s 2024-03-07T03:29:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:02 smithi189 bash[66969]: cluster 2024-03-07T03:29:01.597803+0000 mon.a (mon.0) 1266 : cluster 0 osdmap e745: 8 total, 8 up, 8 in 2024-03-07T03:29:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:02 smithi189 bash[66969]: cluster 2024-03-07T03:29:02.500031+0000 mon.a (mon.0) 1267 : cluster 0 osdmap e746: 8 total, 8 up, 8 in 2024-03-07T03:29:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:03 smithi145 bash[79878]: audit 2024-03-07T03:29:02.652940+0000 mon.a (mon.0) 1268 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:03 smithi145 bash[79878]: cluster 2024-03-07T03:29:03.501901+0000 mon.a (mon.0) 1269 : cluster 0 osdmap e747: 8 total, 8 up, 8 in 2024-03-07T03:29:04.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:03 smithi145 bash[82329]: audit 2024-03-07T03:29:02.652940+0000 mon.a (mon.0) 1268 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:04.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:03 smithi145 bash[82329]: cluster 2024-03-07T03:29:03.501901+0000 mon.a (mon.0) 1269 : cluster 0 osdmap e747: 8 total, 8 up, 8 in 2024-03-07T03:29:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:03 smithi189 bash[66969]: audit 2024-03-07T03:29:02.652940+0000 mon.a (mon.0) 1268 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:03 smithi189 bash[66969]: cluster 2024-03-07T03:29:03.501901+0000 mon.a (mon.0) 1269 : cluster 0 osdmap e747: 8 total, 8 up, 8 in 2024-03-07T03:29:05.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:04 smithi145 bash[79878]: cluster 2024-03-07T03:29:03.076859+0000 mgr.y (mgr.34107) 516 : cluster 0 pgmap v887: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 4.2 KiB/s wr, 80 op/s 2024-03-07T03:29:05.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:04 smithi145 bash[79878]: cluster 2024-03-07T03:29:04.504449+0000 mon.a (mon.0) 1270 : cluster 0 osdmap e748: 8 total, 8 up, 8 in 2024-03-07T03:29:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:04 smithi145 bash[82329]: cluster 2024-03-07T03:29:03.076859+0000 mgr.y (mgr.34107) 516 : cluster 0 pgmap v887: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 4.2 KiB/s wr, 80 op/s 2024-03-07T03:29:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:04 smithi145 bash[82329]: cluster 2024-03-07T03:29:04.504449+0000 mon.a (mon.0) 1270 : cluster 0 osdmap e748: 8 total, 8 up, 8 in 2024-03-07T03:29:05.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:04 smithi189 bash[66969]: cluster 2024-03-07T03:29:03.076859+0000 mgr.y (mgr.34107) 516 : cluster 0 pgmap v887: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 4.2 KiB/s wr, 80 op/s 2024-03-07T03:29:05.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:04 smithi189 bash[66969]: cluster 2024-03-07T03:29:04.504449+0000 mon.a (mon.0) 1270 : cluster 0 osdmap e748: 8 total, 8 up, 8 in 2024-03-07T03:29:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[79878]: audit 2024-03-07T03:29:04.934486+0000 mon.a (mon.0) 1271 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1", "id": [2, 4]}]: dispatch 2024-03-07T03:29:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[79878]: audit 2024-03-07T03:29:04.934958+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]: dispatch 2024-03-07T03:29:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[79878]: audit 2024-03-07T03:29:04.935387+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]: dispatch 2024-03-07T03:29:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[79878]: audit 2024-03-07T03:29:05.501559+0000 mon.a (mon.0) 1274 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1", "id": [2, 4]}]': finished 2024-03-07T03:29:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[79878]: audit 2024-03-07T03:29:05.501785+0000 mon.a (mon.0) 1275 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-03-07T03:29:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[79878]: audit 2024-03-07T03:29:05.501985+0000 mon.a (mon.0) 1276 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]': finished 2024-03-07T03:29:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[79878]: cluster 2024-03-07T03:29:05.509214+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e749: 8 total, 8 up, 8 in 2024-03-07T03:29:06.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[82329]: audit 2024-03-07T03:29:04.934486+0000 mon.a (mon.0) 1271 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1", "id": [2, 4]}]: dispatch 2024-03-07T03:29:06.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[82329]: audit 2024-03-07T03:29:04.934958+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]: dispatch 2024-03-07T03:29:06.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[82329]: audit 2024-03-07T03:29:04.935387+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]: dispatch 2024-03-07T03:29:06.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[82329]: audit 2024-03-07T03:29:05.501559+0000 mon.a (mon.0) 1274 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1", "id": [2, 4]}]': finished 2024-03-07T03:29:06.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[82329]: audit 2024-03-07T03:29:05.501785+0000 mon.a (mon.0) 1275 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-03-07T03:29:06.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[82329]: audit 2024-03-07T03:29:05.501985+0000 mon.a (mon.0) 1276 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]': finished 2024-03-07T03:29:06.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:05 smithi145 bash[82329]: cluster 2024-03-07T03:29:05.509214+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e749: 8 total, 8 up, 8 in 2024-03-07T03:29:06.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:05 smithi189 bash[66969]: audit 2024-03-07T03:29:04.934486+0000 mon.a (mon.0) 1271 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1", "id": [2, 4]}]: dispatch 2024-03-07T03:29:06.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:05 smithi189 bash[66969]: audit 2024-03-07T03:29:04.934958+0000 mon.a (mon.0) 1272 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]: dispatch 2024-03-07T03:29:06.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:05 smithi189 bash[66969]: audit 2024-03-07T03:29:04.935387+0000 mon.a (mon.0) 1273 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]: dispatch 2024-03-07T03:29:06.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:05 smithi189 bash[66969]: audit 2024-03-07T03:29:05.501559+0000 mon.a (mon.0) 1274 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1", "id": [2, 4]}]': finished 2024-03-07T03:29:06.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:05 smithi189 bash[66969]: audit 2024-03-07T03:29:05.501785+0000 mon.a (mon.0) 1275 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-03-07T03:29:06.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:05 smithi189 bash[66969]: audit 2024-03-07T03:29:05.501985+0000 mon.a (mon.0) 1276 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]': finished 2024-03-07T03:29:06.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:05 smithi189 bash[66969]: cluster 2024-03-07T03:29:05.509214+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e749: 8 total, 8 up, 8 in 2024-03-07T03:29:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:06 smithi145 bash[79878]: cluster 2024-03-07T03:29:05.078689+0000 mgr.y (mgr.34107) 517 : cluster 0 pgmap v890: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 8.0 KiB/s wr, 135 op/s 2024-03-07T03:29:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:06 smithi145 bash[79878]: cluster 2024-03-07T03:29:06.516609+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e750: 8 total, 8 up, 8 in 2024-03-07T03:29:07.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:06 smithi145 bash[82329]: cluster 2024-03-07T03:29:05.078689+0000 mgr.y (mgr.34107) 517 : cluster 0 pgmap v890: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 8.0 KiB/s wr, 135 op/s 2024-03-07T03:29:07.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:06 smithi145 bash[82329]: cluster 2024-03-07T03:29:06.516609+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e750: 8 total, 8 up, 8 in 2024-03-07T03:29:07.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:06 smithi189 bash[66969]: cluster 2024-03-07T03:29:05.078689+0000 mgr.y (mgr.34107) 517 : cluster 0 pgmap v890: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 8.0 KiB/s wr, 135 op/s 2024-03-07T03:29:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:06 smithi189 bash[66969]: cluster 2024-03-07T03:29:06.516609+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e750: 8 total, 8 up, 8 in 2024-03-07T03:29:08.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:08 smithi145 bash[79878]: cluster 2024-03-07T03:29:07.081046+0000 mgr.y (mgr.34107) 518 : cluster 0 pgmap v893: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 6.2 KiB/s wr, 123 op/s 2024-03-07T03:29:08.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:08 smithi145 bash[79878]: cluster 2024-03-07T03:29:07.515213+0000 mon.a (mon.0) 1279 : cluster 0 osdmap e751: 8 total, 8 up, 8 in 2024-03-07T03:29:08.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:08 smithi145 bash[82329]: cluster 2024-03-07T03:29:07.081046+0000 mgr.y (mgr.34107) 518 : cluster 0 pgmap v893: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 6.2 KiB/s wr, 123 op/s 2024-03-07T03:29:08.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:08 smithi145 bash[82329]: cluster 2024-03-07T03:29:07.515213+0000 mon.a (mon.0) 1279 : cluster 0 osdmap e751: 8 total, 8 up, 8 in 2024-03-07T03:29:08.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:08 smithi189 bash[66969]: cluster 2024-03-07T03:29:07.081046+0000 mgr.y (mgr.34107) 518 : cluster 0 pgmap v893: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 6.2 KiB/s wr, 123 op/s 2024-03-07T03:29:08.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:08 smithi189 bash[66969]: cluster 2024-03-07T03:29:07.515213+0000 mon.a (mon.0) 1279 : cluster 0 osdmap e751: 8 total, 8 up, 8 in 2024-03-07T03:29:09.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:09 smithi145 bash[79878]: cluster 2024-03-07T03:29:08.528276+0000 mon.a (mon.0) 1280 : cluster 0 osdmap e752: 8 total, 8 up, 8 in 2024-03-07T03:29:09.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:09 smithi145 bash[82329]: cluster 2024-03-07T03:29:08.528276+0000 mon.a (mon.0) 1280 : cluster 0 osdmap e752: 8 total, 8 up, 8 in 2024-03-07T03:29:09.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:09 smithi189 bash[66969]: cluster 2024-03-07T03:29:08.528276+0000 mon.a (mon.0) 1280 : cluster 0 osdmap e752: 8 total, 8 up, 8 in 2024-03-07T03:29:10.144 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ObjectMapConsistentSnap (11138 ms) 2024-03-07T03:29:10.144 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.Metadata 2024-03-07T03:29:10.144 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:10.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:10 smithi145 bash[79878]: cluster 2024-03-07T03:29:09.083936+0000 mgr.y (mgr.34107) 519 : cluster 0 pgmap v896: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 115 KiB/s rd, 20 KiB/s wr, 166 op/s 2024-03-07T03:29:10.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:10 smithi145 bash[82329]: cluster 2024-03-07T03:29:09.083936+0000 mgr.y (mgr.34107) 519 : cluster 0 pgmap v896: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 115 KiB/s rd, 20 KiB/s wr, 166 op/s 2024-03-07T03:29:10.819 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.Metadata (675 ms) 2024-03-07T03:29:10.819 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.MetadataPP 2024-03-07T03:29:10.819 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:10.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:10 smithi189 bash[66969]: cluster 2024-03-07T03:29:09.083936+0000 mgr.y (mgr.34107) 519 : cluster 0 pgmap v896: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 115 KiB/s rd, 20 KiB/s wr, 166 op/s 2024-03-07T03:29:11.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:11 smithi145 bash[79878]: cluster 2024-03-07T03:29:10.543559+0000 mon.a (mon.0) 1281 : cluster 0 osdmap e753: 8 total, 8 up, 8 in 2024-03-07T03:29:11.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:11 smithi145 bash[82329]: cluster 2024-03-07T03:29:10.543559+0000 mon.a (mon.0) 1281 : cluster 0 osdmap e753: 8 total, 8 up, 8 in 2024-03-07T03:29:11.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:11 smithi189 bash[66969]: cluster 2024-03-07T03:29:10.543559+0000 mon.a (mon.0) 1281 : cluster 0 osdmap e753: 8 total, 8 up, 8 in 2024-03-07T03:29:11.849 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.MetadataPP (1031 ms) 2024-03-07T03:29:11.849 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.UpdateFeatures 2024-03-07T03:29:11.850 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:12.134 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.UpdateFeatures (284 ms) 2024-03-07T03:29:12.134 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.FeaturesBitmaskString 2024-03-07T03:29:12.134 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.FeaturesBitmaskString (0 ms) 2024-03-07T03:29:12.134 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.RebuildObjectMap 2024-03-07T03:29:12.134 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:12.556 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:29:12 smithi189 bash[59814]: ts=2024-03-07T03:29:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:29:12.557 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:12 smithi189 bash[66969]: cluster 2024-03-07T03:29:11.086353+0000 mgr.y (mgr.34107) 520 : cluster 0 pgmap v898: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 35 KiB/s wr, 170 op/s 2024-03-07T03:29:12.746 INFO:tasks.workunit.client.0.smithi145.stdout:0.00% done 2024-03-07T03:29:12.746 INFO:tasks.workunit.client.0.smithi145.stdout:25.00% done 2024-03-07T03:29:12.746 INFO:tasks.workunit.client.0.smithi145.stdout:50.00% done 2024-03-07T03:29:12.746 INFO:tasks.workunit.client.0.smithi145.stdout:75.00% done 2024-03-07T03:29:12.746 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.RebuildObjectMap (612 ms) 2024-03-07T03:29:12.746 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.RebuildNewObjectMap 2024-03-07T03:29:12.808 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:29:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:29:12] "GET /metrics HTTP/1.1" 200 75033 "" "Prometheus/2.43.0" 2024-03-07T03:29:12.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:12 smithi145 bash[79878]: cluster 2024-03-07T03:29:11.086353+0000 mgr.y (mgr.34107) 520 : cluster 0 pgmap v898: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 35 KiB/s wr, 170 op/s 2024-03-07T03:29:12.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:12 smithi145 bash[79878]: cluster 2024-03-07T03:29:11.555616+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e754: 8 total, 8 up, 8 in 2024-03-07T03:29:12.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:12 smithi145 bash[82329]: cluster 2024-03-07T03:29:11.086353+0000 mgr.y (mgr.34107) 520 : cluster 0 pgmap v898: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 35 KiB/s wr, 170 op/s 2024-03-07T03:29:12.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:12 smithi145 bash[82329]: cluster 2024-03-07T03:29:11.555616+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e754: 8 total, 8 up, 8 in 2024-03-07T03:29:12.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:12 smithi189 bash[66969]: cluster 2024-03-07T03:29:11.555616+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e754: 8 total, 8 up, 8 in 2024-03-07T03:29:12.835 INFO:tasks.workunit.client.0.smithi145.stdout:0.00% done 2024-03-07T03:29:12.835 INFO:tasks.workunit.client.0.smithi145.stdout:25.00% done 2024-03-07T03:29:12.835 INFO:tasks.workunit.client.0.smithi145.stdout:50.00% done 2024-03-07T03:29:12.835 INFO:tasks.workunit.client.0.smithi145.stdout:75.00% done 2024-03-07T03:29:12.835 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.RebuildNewObjectMap (89 ms) 2024-03-07T03:29:12.835 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.CheckObjectMap 2024-03-07T03:29:12.835 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:13.749 INFO:tasks.workunit.client.0.smithi145.stdout:0.00% done 2024-03-07T03:29:13.749 INFO:tasks.workunit.client.0.smithi145.stdout:25.00% done 2024-03-07T03:29:13.749 INFO:tasks.workunit.client.0.smithi145.stdout:50.00% done 2024-03-07T03:29:13.749 INFO:tasks.workunit.client.0.smithi145.stdout:75.00% done 2024-03-07T03:29:13.749 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.CheckObjectMap (914 ms) 2024-03-07T03:29:13.749 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.BlockingAIO 2024-03-07T03:29:13.750 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:13.824 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.BlockingAIO (76 ms) 2024-03-07T03:29:13.825 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ExclusiveLockTransition 2024-03-07T03:29:13.825 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:13.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:13 smithi189 bash[66969]: cluster 2024-03-07T03:29:12.555960+0000 mon.a (mon.0) 1283 : cluster 0 osdmap e755: 8 total, 8 up, 8 in 2024-03-07T03:29:14.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:13 smithi145 bash[79878]: cluster 2024-03-07T03:29:12.555960+0000 mon.a (mon.0) 1283 : cluster 0 osdmap e755: 8 total, 8 up, 8 in 2024-03-07T03:29:14.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:13 smithi145 bash[82329]: cluster 2024-03-07T03:29:12.555960+0000 mon.a (mon.0) 1283 : cluster 0 osdmap e755: 8 total, 8 up, 8 in 2024-03-07T03:29:14.082 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ExclusiveLockTransition (257 ms) 2024-03-07T03:29:14.082 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ExclusiveLockReadTransition 2024-03-07T03:29:14.082 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_librbd.cc:10118: Skipped 2024-03-07T03:29:14.082 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:29:14.082 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestLibRBD.ExclusiveLockReadTransition (0 ms) 2024-03-07T03:29:14.082 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.CacheMayCopyOnWrite 2024-03-07T03:29:14.082 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:14.766 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.CacheMayCopyOnWrite (684 ms) 2024-03-07T03:29:14.766 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot 2024-03-07T03:29:14.766 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:14.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:14 smithi189 bash[66969]: cluster 2024-03-07T03:29:13.087413+0000 mgr.y (mgr.34107) 521 : cluster 0 pgmap v901: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 109 KiB/s rd, 49 KiB/s wr, 189 op/s 2024-03-07T03:29:14.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:14 smithi189 bash[66969]: cluster 2024-03-07T03:29:13.583546+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e756: 8 total, 8 up, 8 in 2024-03-07T03:29:15.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:14 smithi145 bash[79878]: cluster 2024-03-07T03:29:13.087413+0000 mgr.y (mgr.34107) 521 : cluster 0 pgmap v901: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 109 KiB/s rd, 49 KiB/s wr, 189 op/s 2024-03-07T03:29:15.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:14 smithi145 bash[79878]: cluster 2024-03-07T03:29:13.583546+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e756: 8 total, 8 up, 8 in 2024-03-07T03:29:15.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:14 smithi145 bash[82329]: cluster 2024-03-07T03:29:13.087413+0000 mgr.y (mgr.34107) 521 : cluster 0 pgmap v901: 617 pgs: 617 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 109 KiB/s rd, 49 KiB/s wr, 189 op/s 2024-03-07T03:29:15.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:14 smithi145 bash[82329]: cluster 2024-03-07T03:29:13.583546+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e756: 8 total, 8 up, 8 in 2024-03-07T03:29:15.687 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot (921 ms) 2024-03-07T03:29:15.687 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestImageOptions 2024-03-07T03:29:16.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:15 smithi145 bash[79878]: cluster 2024-03-07T03:29:14.580129+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e757: 8 total, 8 up, 8 in 2024-03-07T03:29:16.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:15 smithi145 bash[82329]: cluster 2024-03-07T03:29:14.580129+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e757: 8 total, 8 up, 8 in 2024-03-07T03:29:16.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:15 smithi189 bash[66969]: cluster 2024-03-07T03:29:14.580129+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e757: 8 total, 8 up, 8 in 2024-03-07T03:29:16.960 INFO:tasks.workunit.client.0.smithi145.stdout:0.00% done 2024-03-07T03:29:16.960 INFO:tasks.workunit.client.0.smithi145.stdout:100.00% done 2024-03-07T03:29:16.960 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestImageOptions (1273 ms) 2024-03-07T03:29:16.960 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestImageOptionsPP 2024-03-07T03:29:17.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:16 smithi145 bash[79878]: cluster 2024-03-07T03:29:15.089065+0000 mgr.y (mgr.34107) 522 : cluster 0 pgmap v904: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 562 KiB/s rd, 187 KiB/s wr, 867 op/s 2024-03-07T03:29:17.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:16 smithi145 bash[79878]: cluster 2024-03-07T03:29:15.608541+0000 mon.a (mon.0) 1286 : cluster 0 osdmap e758: 8 total, 8 up, 8 in 2024-03-07T03:29:17.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:16 smithi145 bash[82329]: cluster 2024-03-07T03:29:15.089065+0000 mgr.y (mgr.34107) 522 : cluster 0 pgmap v904: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 562 KiB/s rd, 187 KiB/s wr, 867 op/s 2024-03-07T03:29:17.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:16 smithi145 bash[82329]: cluster 2024-03-07T03:29:15.608541+0000 mon.a (mon.0) 1286 : cluster 0 osdmap e758: 8 total, 8 up, 8 in 2024-03-07T03:29:17.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:16 smithi189 bash[66969]: cluster 2024-03-07T03:29:15.089065+0000 mgr.y (mgr.34107) 522 : cluster 0 pgmap v904: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 562 KiB/s rd, 187 KiB/s wr, 867 op/s 2024-03-07T03:29:17.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:16 smithi189 bash[66969]: cluster 2024-03-07T03:29:15.608541+0000 mon.a (mon.0) 1286 : cluster 0 osdmap e758: 8 total, 8 up, 8 in 2024-03-07T03:29:17.907 INFO:tasks.workunit.client.0.smithi145.stdout:0.00% done 2024-03-07T03:29:17.908 INFO:tasks.workunit.client.0.smithi145.stdout:100.00% done 2024-03-07T03:29:17.908 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestImageOptionsPP (947 ms) 2024-03-07T03:29:17.908 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.EventSocketPipe 2024-03-07T03:29:17.908 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.EventSocketPipe (0 ms) 2024-03-07T03:29:17.908 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.EventSocketEventfd 2024-03-07T03:29:17.908 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.EventSocketEventfd (0 ms) 2024-03-07T03:29:17.908 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ImagePollIO 2024-03-07T03:29:17.908 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:18.007 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:29:18.008 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:29:18.008 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:29:18.008 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:29:18.008 INFO:tasks.workunit.client.0.smithi145.stdout:wrote: 512 2024-03-07T03:29:18.008 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:29:18.008 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:29:18.008 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:29:18.008 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:29:18.008 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:29:18.008 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:29:18.008 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:29:18.008 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:29:18.008 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:started write 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:write completion cb called! 2024-03-07T03:29:18.009 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 0 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:finished write 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:29:18.010 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:29:18.011 INFO:tasks.workunit.client.0.smithi145.stdout:created completion 2024-03-07T03:29:18.011 INFO:tasks.workunit.client.0.smithi145.stdout:started read 2024-03-07T03:29:18.011 INFO:tasks.workunit.client.0.smithi145.stdout:read completion cb called! 2024-03-07T03:29:18.011 INFO:tasks.workunit.client.0.smithi145.stdout:return value is: 512 2024-03-07T03:29:18.011 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ImagePollIO (101 ms) 2024-03-07T03:29:18.011 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.Mirror 2024-03-07T03:29:18.019 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:18.038 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:18.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:17 smithi145 bash[82329]: cluster 2024-03-07T03:29:16.624512+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e759: 8 total, 8 up, 8 in 2024-03-07T03:29:18.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:17 smithi145 bash[79878]: cluster 2024-03-07T03:29:16.624512+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e759: 8 total, 8 up, 8 in 2024-03-07T03:29:18.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:17 smithi189 bash[66969]: cluster 2024-03-07T03:29:16.624512+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e759: 8 total, 8 up, 8 in 2024-03-07T03:29:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:18 smithi145 bash[79878]: cluster 2024-03-07T03:29:17.091474+0000 mgr.y (mgr.34107) 523 : cluster 0 pgmap v907: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 739 KiB/s rd, 213 KiB/s wr, 1.06k op/s 2024-03-07T03:29:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:18 smithi145 bash[79878]: cluster 2024-03-07T03:29:17.631760+0000 mon.a (mon.0) 1288 : cluster 0 osdmap e760: 8 total, 8 up, 8 in 2024-03-07T03:29:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:18 smithi145 bash[79878]: audit 2024-03-07T03:29:17.654338+0000 mon.a (mon.0) 1289 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:19.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:18 smithi145 bash[82329]: cluster 2024-03-07T03:29:17.091474+0000 mgr.y (mgr.34107) 523 : cluster 0 pgmap v907: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 739 KiB/s rd, 213 KiB/s wr, 1.06k op/s 2024-03-07T03:29:19.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:18 smithi145 bash[82329]: cluster 2024-03-07T03:29:17.631760+0000 mon.a (mon.0) 1288 : cluster 0 osdmap e760: 8 total, 8 up, 8 in 2024-03-07T03:29:19.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:18 smithi145 bash[82329]: audit 2024-03-07T03:29:17.654338+0000 mon.a (mon.0) 1289 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:19.070 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.Mirror (1062 ms) 2024-03-07T03:29:19.070 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.MirrorPeerAttributes 2024-03-07T03:29:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:18 smithi189 bash[66969]: cluster 2024-03-07T03:29:17.091474+0000 mgr.y (mgr.34107) 523 : cluster 0 pgmap v907: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 739 KiB/s rd, 213 KiB/s wr, 1.06k op/s 2024-03-07T03:29:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:18 smithi189 bash[66969]: cluster 2024-03-07T03:29:17.631760+0000 mon.a (mon.0) 1288 : cluster 0 osdmap e760: 8 total, 8 up, 8 in 2024-03-07T03:29:19.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:18 smithi189 bash[66969]: audit 2024-03-07T03:29:17.654338+0000 mon.a (mon.0) 1289 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:19.114 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.MirrorPeerAttributes (44 ms) 2024-03-07T03:29:19.114 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.CreateWithMirrorEnabled 2024-03-07T03:29:20.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: cluster 2024-03-07T03:29:18.642004+0000 mon.a (mon.0) 1290 : cluster 0 osdmap e761: 8 total, 8 up, 8 in 2024-03-07T03:29:20.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.009829+0000 mon.c (mon.2) 38 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-07T03:29:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.010611+0000 mon.a (mon.0) 1291 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-07T03:29:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.017150+0000 mon.a (mon.0) 1292 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-03-07T03:29:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.022805+0000 mon.c (mon.2) 39 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/0f3f6bf0-1a31-450c-a8a4-08fb6735ea04"}]: dispatch 2024-03-07T03:29:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.023455+0000 mon.a (mon.0) 1293 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/0f3f6bf0-1a31-450c-a8a4-08fb6735ea04"}]: dispatch 2024-03-07T03:29:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.029357+0000 mon.a (mon.0) 1294 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/0f3f6bf0-1a31-450c-a8a4-08fb6735ea04"}]': finished 2024-03-07T03:29:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.048519+0000 mon.c (mon.2) 40 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/2befdc52-d904-4035-9b06-fc942f6a495a"}]: dispatch 2024-03-07T03:29:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.049178+0000 mon.a (mon.0) 1295 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/2befdc52-d904-4035-9b06-fc942f6a495a"}]: dispatch 2024-03-07T03:29:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.055120+0000 mon.a (mon.0) 1296 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/2befdc52-d904-4035-9b06-fc942f6a495a"}]': finished 2024-03-07T03:29:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.061818+0000 mon.c (mon.2) 41 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4750440d-b2db-4605-9b4c-9c15b6c9b935"}]: dispatch 2024-03-07T03:29:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.062642+0000 mon.a (mon.0) 1297 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4750440d-b2db-4605-9b4c-9c15b6c9b935"}]: dispatch 2024-03-07T03:29:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.068567+0000 mon.a (mon.0) 1298 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4750440d-b2db-4605-9b4c-9c15b6c9b935"}]': finished 2024-03-07T03:29:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.094584+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]: dispatch 2024-03-07T03:29:20.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.104769+0000 mon.a (mon.0) 1299 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:29:20.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.106499+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]: dispatch 2024-03-07T03:29:20.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: cluster 2024-03-07T03:29:18.642004+0000 mon.a (mon.0) 1290 : cluster 0 osdmap e761: 8 total, 8 up, 8 in 2024-03-07T03:29:20.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.009829+0000 mon.c (mon.2) 38 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-07T03:29:20.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.010611+0000 mon.a (mon.0) 1291 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-07T03:29:20.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.017150+0000 mon.a (mon.0) 1292 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-03-07T03:29:20.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.022805+0000 mon.c (mon.2) 39 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/0f3f6bf0-1a31-450c-a8a4-08fb6735ea04"}]: dispatch 2024-03-07T03:29:20.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.023455+0000 mon.a (mon.0) 1293 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/0f3f6bf0-1a31-450c-a8a4-08fb6735ea04"}]: dispatch 2024-03-07T03:29:20.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.029357+0000 mon.a (mon.0) 1294 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/0f3f6bf0-1a31-450c-a8a4-08fb6735ea04"}]': finished 2024-03-07T03:29:20.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.048519+0000 mon.c (mon.2) 40 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/2befdc52-d904-4035-9b06-fc942f6a495a"}]: dispatch 2024-03-07T03:29:20.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.049178+0000 mon.a (mon.0) 1295 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/2befdc52-d904-4035-9b06-fc942f6a495a"}]: dispatch 2024-03-07T03:29:20.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.055120+0000 mon.a (mon.0) 1296 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/2befdc52-d904-4035-9b06-fc942f6a495a"}]': finished 2024-03-07T03:29:20.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.061818+0000 mon.c (mon.2) 41 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4750440d-b2db-4605-9b4c-9c15b6c9b935"}]: dispatch 2024-03-07T03:29:20.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.062642+0000 mon.a (mon.0) 1297 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4750440d-b2db-4605-9b4c-9c15b6c9b935"}]: dispatch 2024-03-07T03:29:20.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.068567+0000 mon.a (mon.0) 1298 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4750440d-b2db-4605-9b4c-9c15b6c9b935"}]': finished 2024-03-07T03:29:20.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.094584+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]: dispatch 2024-03-07T03:29:20.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.104769+0000 mon.a (mon.0) 1299 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:29:20.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.106499+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]: dispatch 2024-03-07T03:29:20.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.107381+0000 mon.c (mon.2) 44 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]: dispatch 2024-03-07T03:29:20.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.108062+0000 mon.a (mon.0) 1300 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]: dispatch 2024-03-07T03:29:20.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[79878]: audit 2024-03-07T03:29:19.114240+0000 mon.a (mon.0) 1301 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]': finished 2024-03-07T03:29:20.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.107381+0000 mon.c (mon.2) 44 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]: dispatch 2024-03-07T03:29:20.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.108062+0000 mon.a (mon.0) 1300 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]: dispatch 2024-03-07T03:29:20.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:19 smithi145 bash[82329]: audit 2024-03-07T03:29:19.114240+0000 mon.a (mon.0) 1301 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]': finished 2024-03-07T03:29:20.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: cluster 2024-03-07T03:29:18.642004+0000 mon.a (mon.0) 1290 : cluster 0 osdmap e761: 8 total, 8 up, 8 in 2024-03-07T03:29:20.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.009829+0000 mon.c (mon.2) 38 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-07T03:29:20.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.010611+0000 mon.a (mon.0) 1291 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-07T03:29:20.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.017150+0000 mon.a (mon.0) 1292 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-03-07T03:29:20.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.022805+0000 mon.c (mon.2) 39 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/0f3f6bf0-1a31-450c-a8a4-08fb6735ea04"}]: dispatch 2024-03-07T03:29:20.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.023455+0000 mon.a (mon.0) 1293 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/0f3f6bf0-1a31-450c-a8a4-08fb6735ea04"}]: dispatch 2024-03-07T03:29:20.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.029357+0000 mon.a (mon.0) 1294 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/0f3f6bf0-1a31-450c-a8a4-08fb6735ea04"}]': finished 2024-03-07T03:29:20.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.048519+0000 mon.c (mon.2) 40 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/2befdc52-d904-4035-9b06-fc942f6a495a"}]: dispatch 2024-03-07T03:29:20.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.049178+0000 mon.a (mon.0) 1295 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/2befdc52-d904-4035-9b06-fc942f6a495a"}]: dispatch 2024-03-07T03:29:20.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.055120+0000 mon.a (mon.0) 1296 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/2befdc52-d904-4035-9b06-fc942f6a495a"}]': finished 2024-03-07T03:29:20.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.061818+0000 mon.c (mon.2) 41 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4750440d-b2db-4605-9b4c-9c15b6c9b935"}]: dispatch 2024-03-07T03:29:20.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.062642+0000 mon.a (mon.0) 1297 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4750440d-b2db-4605-9b4c-9c15b6c9b935"}]: dispatch 2024-03-07T03:29:20.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.068567+0000 mon.a (mon.0) 1298 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4750440d-b2db-4605-9b4c-9c15b6c9b935"}]': finished 2024-03-07T03:29:20.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.094584+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]: dispatch 2024-03-07T03:29:20.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.104769+0000 mon.a (mon.0) 1299 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:29:20.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.106499+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]: dispatch 2024-03-07T03:29:20.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.107381+0000 mon.c (mon.2) 44 : audit 1 from='client.? 172.21.15.145:0/3163392695' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]: dispatch 2024-03-07T03:29:20.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.108062+0000 mon.a (mon.0) 1300 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]: dispatch 2024-03-07T03:29:20.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:19 smithi189 bash[66969]: audit 2024-03-07T03:29:19.114240+0000 mon.a (mon.0) 1301 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4c446ca0-5141-4699-b850-57ac84356170"}]': finished 2024-03-07T03:29:21.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:20 smithi145 bash[79878]: cluster 2024-03-07T03:29:19.094176+0000 mgr.y (mgr.34107) 524 : cluster 0 pgmap v910: 617 pgs: 617 active+clean; 310 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 335 KiB/s rd, 1.4 MiB/s wr, 441 op/s 2024-03-07T03:29:21.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:20 smithi145 bash[79878]: cluster 2024-03-07T03:29:20.122834+0000 mon.a (mon.0) 1302 : cluster 0 osdmap e762: 8 total, 8 up, 8 in 2024-03-07T03:29:21.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:20 smithi145 bash[82329]: cluster 2024-03-07T03:29:19.094176+0000 mgr.y (mgr.34107) 524 : cluster 0 pgmap v910: 617 pgs: 617 active+clean; 310 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 335 KiB/s rd, 1.4 MiB/s wr, 441 op/s 2024-03-07T03:29:21.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:20 smithi145 bash[82329]: cluster 2024-03-07T03:29:20.122834+0000 mon.a (mon.0) 1302 : cluster 0 osdmap e762: 8 total, 8 up, 8 in 2024-03-07T03:29:21.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:20 smithi189 bash[66969]: cluster 2024-03-07T03:29:19.094176+0000 mgr.y (mgr.34107) 524 : cluster 0 pgmap v910: 617 pgs: 617 active+clean; 310 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 335 KiB/s rd, 1.4 MiB/s wr, 441 op/s 2024-03-07T03:29:21.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:20 smithi189 bash[66969]: cluster 2024-03-07T03:29:20.122834+0000 mon.a (mon.0) 1302 : cluster 0 osdmap e762: 8 total, 8 up, 8 in 2024-03-07T03:29:22.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:22 smithi145 bash[79878]: cluster 2024-03-07T03:29:21.096441+0000 mgr.y (mgr.34107) 525 : cluster 0 pgmap v912: 617 pgs: 617 active+clean; 312 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 363 KiB/s rd, 1.7 MiB/s wr, 494 op/s 2024-03-07T03:29:22.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:22 smithi145 bash[79878]: cluster 2024-03-07T03:29:21.127628+0000 mon.a (mon.0) 1303 : cluster 0 osdmap e763: 8 total, 8 up, 8 in 2024-03-07T03:29:22.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:22 smithi145 bash[82329]: cluster 2024-03-07T03:29:21.096441+0000 mgr.y (mgr.34107) 525 : cluster 0 pgmap v912: 617 pgs: 617 active+clean; 312 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 363 KiB/s rd, 1.7 MiB/s wr, 494 op/s 2024-03-07T03:29:22.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:22 smithi145 bash[82329]: cluster 2024-03-07T03:29:21.127628+0000 mon.a (mon.0) 1303 : cluster 0 osdmap e763: 8 total, 8 up, 8 in 2024-03-07T03:29:22.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:29:22 smithi189 bash[59814]: ts=2024-03-07T03:29:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:29:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:22 smithi189 bash[66969]: cluster 2024-03-07T03:29:21.096441+0000 mgr.y (mgr.34107) 525 : cluster 0 pgmap v912: 617 pgs: 617 active+clean; 312 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 363 KiB/s rd, 1.7 MiB/s wr, 494 op/s 2024-03-07T03:29:22.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:22 smithi189 bash[66969]: cluster 2024-03-07T03:29:21.127628+0000 mon.a (mon.0) 1303 : cluster 0 osdmap e763: 8 total, 8 up, 8 in 2024-03-07T03:29:23.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:29:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:29:22] "GET /metrics HTTP/1.1" 200 75027 "" "Prometheus/2.43.0" 2024-03-07T03:29:23.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:23 smithi145 bash[79878]: cluster 2024-03-07T03:29:22.129482+0000 mon.a (mon.0) 1304 : cluster 0 osdmap e764: 8 total, 8 up, 8 in 2024-03-07T03:29:23.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:23 smithi145 bash[82329]: cluster 2024-03-07T03:29:22.129482+0000 mon.a (mon.0) 1304 : cluster 0 osdmap e764: 8 total, 8 up, 8 in 2024-03-07T03:29:23.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:23 smithi189 bash[66969]: cluster 2024-03-07T03:29:22.129482+0000 mon.a (mon.0) 1304 : cluster 0 osdmap e764: 8 total, 8 up, 8 in 2024-03-07T03:29:24.229 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.CreateWithMirrorEnabled (5116 ms) 2024-03-07T03:29:24.230 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot 2024-03-07T03:29:24.230 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:24.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:24 smithi145 bash[79878]: cluster 2024-03-07T03:29:23.097544+0000 mgr.y (mgr.34107) 526 : cluster 0 pgmap v915: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 298 KiB/s rd, 1.9 MiB/s wr, 412 op/s 2024-03-07T03:29:24.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:24 smithi145 bash[79878]: cluster 2024-03-07T03:29:23.157908+0000 mon.a (mon.0) 1305 : cluster 0 osdmap e765: 8 total, 8 up, 8 in 2024-03-07T03:29:24.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:24 smithi145 bash[79878]: audit 2024-03-07T03:29:23.176973+0000 mon.a (mon.0) 1306 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:29:24.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:24 smithi145 bash[79878]: audit 2024-03-07T03:29:23.704850+0000 mon.a (mon.0) 1307 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:24.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:24 smithi145 bash[79878]: audit 2024-03-07T03:29:23.714408+0000 mon.a (mon.0) 1308 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:24.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:24 smithi145 bash[82329]: cluster 2024-03-07T03:29:23.097544+0000 mgr.y (mgr.34107) 526 : cluster 0 pgmap v915: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 298 KiB/s rd, 1.9 MiB/s wr, 412 op/s 2024-03-07T03:29:24.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:24 smithi145 bash[82329]: cluster 2024-03-07T03:29:23.157908+0000 mon.a (mon.0) 1305 : cluster 0 osdmap e765: 8 total, 8 up, 8 in 2024-03-07T03:29:24.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:24 smithi145 bash[82329]: audit 2024-03-07T03:29:23.176973+0000 mon.a (mon.0) 1306 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:29:24.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:24 smithi145 bash[82329]: audit 2024-03-07T03:29:23.704850+0000 mon.a (mon.0) 1307 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:24.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:24 smithi145 bash[82329]: audit 2024-03-07T03:29:23.714408+0000 mon.a (mon.0) 1308 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:24.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:24 smithi189 bash[66969]: cluster 2024-03-07T03:29:23.097544+0000 mgr.y (mgr.34107) 526 : cluster 0 pgmap v915: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 298 KiB/s rd, 1.9 MiB/s wr, 412 op/s 2024-03-07T03:29:24.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:24 smithi189 bash[66969]: cluster 2024-03-07T03:29:23.157908+0000 mon.a (mon.0) 1305 : cluster 0 osdmap e765: 8 total, 8 up, 8 in 2024-03-07T03:29:24.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:24 smithi189 bash[66969]: audit 2024-03-07T03:29:23.176973+0000 mon.a (mon.0) 1306 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:29:24.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:24 smithi189 bash[66969]: audit 2024-03-07T03:29:23.704850+0000 mon.a (mon.0) 1307 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:24.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:24 smithi189 bash[66969]: audit 2024-03-07T03:29:23.714408+0000 mon.a (mon.0) 1308 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:25.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:25 smithi145 bash[79878]: cluster 2024-03-07T03:29:24.166462+0000 mon.a (mon.0) 1309 : cluster 0 osdmap e766: 8 total, 8 up, 8 in 2024-03-07T03:29:25.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:25 smithi145 bash[82329]: cluster 2024-03-07T03:29:24.166462+0000 mon.a (mon.0) 1309 : cluster 0 osdmap e766: 8 total, 8 up, 8 in 2024-03-07T03:29:25.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:25 smithi189 bash[66969]: cluster 2024-03-07T03:29:24.166462+0000 mon.a (mon.0) 1309 : cluster 0 osdmap e766: 8 total, 8 up, 8 in 2024-03-07T03:29:26.211 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot (1982 ms) 2024-03-07T03:29:26.212 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ExclusiveLock 2024-03-07T03:29:26.212 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:26.315 INFO:tasks.workunit.client.0.smithi145.stdout:1: acquiring exclusive lock 2024-03-07T03:29:26.315 INFO:tasks.workunit.client.0.smithi145.stdout:2: acquiring exclusive lock 2024-03-07T03:29:26.322 INFO:tasks.workunit.client.0.smithi145.stdout:2: exclusive lock acquired 2024-03-07T03:29:26.353 INFO:tasks.workunit.client.0.smithi145.stdout:2: releasing exclusive lock 2024-03-07T03:29:26.359 INFO:tasks.workunit.client.0.smithi145.stdout:2: exclusive lock released 2024-03-07T03:29:26.359 INFO:tasks.workunit.client.0.smithi145.stdout:2: acquiring exclusive lock 2024-03-07T03:29:26.369 INFO:tasks.workunit.client.0.smithi145.stdout:1: exclusive lock acquired 2024-03-07T03:29:26.378 INFO:tasks.workunit.client.0.smithi145.stdout:1: releasing exclusive lock 2024-03-07T03:29:26.384 INFO:tasks.workunit.client.0.smithi145.stdout:1: exclusive lock released 2024-03-07T03:29:26.384 INFO:tasks.workunit.client.0.smithi145.stdout:1: acquiring exclusive lock 2024-03-07T03:29:26.390 INFO:tasks.workunit.client.0.smithi145.stdout:2: exclusive lock acquired 2024-03-07T03:29:26.419 INFO:tasks.workunit.client.0.smithi145.stdout:2: releasing exclusive lock 2024-03-07T03:29:26.426 INFO:tasks.workunit.client.0.smithi145.stdout:2: exclusive lock released 2024-03-07T03:29:26.426 INFO:tasks.workunit.client.0.smithi145.stdout:2: acquiring exclusive lock 2024-03-07T03:29:26.435 INFO:tasks.workunit.client.0.smithi145.stdout:1: exclusive lock acquired 2024-03-07T03:29:26.441 INFO:tasks.workunit.client.0.smithi145.stdout:1: releasing exclusive lock 2024-03-07T03:29:26.447 INFO:tasks.workunit.client.0.smithi145.stdout:1: exclusive lock released 2024-03-07T03:29:26.447 INFO:tasks.workunit.client.0.smithi145.stdout:1: acquiring exclusive lock 2024-03-07T03:29:26.455 INFO:tasks.workunit.client.0.smithi145.stdout:1: exclusive lock acquired 2024-03-07T03:29:26.495 INFO:tasks.workunit.client.0.smithi145.stdout:1: releasing exclusive lock 2024-03-07T03:29:26.502 INFO:tasks.workunit.client.0.smithi145.stdout:1: exclusive lock released 2024-03-07T03:29:26.502 INFO:tasks.workunit.client.0.smithi145.stdout:1: acquiring exclusive lock 2024-03-07T03:29:26.510 INFO:tasks.workunit.client.0.smithi145.stdout:2: exclusive lock acquired 2024-03-07T03:29:26.548 INFO:tasks.workunit.client.0.smithi145.stdout:2: releasing exclusive lock 2024-03-07T03:29:26.554 INFO:tasks.workunit.client.0.smithi145.stdout:2: exclusive lock released 2024-03-07T03:29:26.555 INFO:tasks.workunit.client.0.smithi145.stdout:2: acquiring exclusive lock 2024-03-07T03:29:26.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:26 smithi145 bash[79878]: cluster 2024-03-07T03:29:25.099309+0000 mgr.y (mgr.34107) 527 : cluster 0 pgmap v918: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 174 KiB/s rd, 269 KiB/s wr, 225 op/s 2024-03-07T03:29:26.560 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:26 smithi145 bash[79878]: cluster 2024-03-07T03:29:25.178240+0000 mon.a (mon.0) 1310 : cluster 0 osdmap e767: 8 total, 8 up, 8 in 2024-03-07T03:29:26.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:26 smithi145 bash[82329]: cluster 2024-03-07T03:29:25.099309+0000 mgr.y (mgr.34107) 527 : cluster 0 pgmap v918: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 174 KiB/s rd, 269 KiB/s wr, 225 op/s 2024-03-07T03:29:26.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:26 smithi145 bash[82329]: cluster 2024-03-07T03:29:25.178240+0000 mon.a (mon.0) 1310 : cluster 0 osdmap e767: 8 total, 8 up, 8 in 2024-03-07T03:29:26.565 INFO:tasks.workunit.client.0.smithi145.stdout:2: exclusive lock acquired 2024-03-07T03:29:26.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:26 smithi189 bash[66969]: cluster 2024-03-07T03:29:25.099309+0000 mgr.y (mgr.34107) 527 : cluster 0 pgmap v918: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 174 KiB/s rd, 269 KiB/s wr, 225 op/s 2024-03-07T03:29:26.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:26 smithi189 bash[66969]: cluster 2024-03-07T03:29:25.178240+0000 mon.a (mon.0) 1310 : cluster 0 osdmap e767: 8 total, 8 up, 8 in 2024-03-07T03:29:26.586 INFO:tasks.workunit.client.0.smithi145.stdout:2: releasing exclusive lock 2024-03-07T03:29:26.594 INFO:tasks.workunit.client.0.smithi145.stdout:2: exclusive lock released 2024-03-07T03:29:26.594 INFO:tasks.workunit.client.0.smithi145.stdout:2: acquiring exclusive lock 2024-03-07T03:29:26.603 INFO:tasks.workunit.client.0.smithi145.stdout:1: exclusive lock acquired 2024-03-07T03:29:26.603 INFO:tasks.workunit.client.0.smithi145.stdout:1: releasing exclusive lock 2024-03-07T03:29:26.611 INFO:tasks.workunit.client.0.smithi145.stdout:1: exclusive lock released 2024-03-07T03:29:26.611 INFO:tasks.workunit.client.0.smithi145.stdout:1: acquiring exclusive lock 2024-03-07T03:29:26.620 INFO:tasks.workunit.client.0.smithi145.stdout:2: exclusive lock acquired 2024-03-07T03:29:26.665 INFO:tasks.workunit.client.0.smithi145.stdout:2: releasing exclusive lock 2024-03-07T03:29:26.671 INFO:tasks.workunit.client.0.smithi145.stdout:2: exclusive lock released 2024-03-07T03:29:26.679 INFO:tasks.workunit.client.0.smithi145.stdout:1: exclusive lock acquired 2024-03-07T03:29:26.707 INFO:tasks.workunit.client.0.smithi145.stdout:1: releasing exclusive lock 2024-03-07T03:29:26.712 INFO:tasks.workunit.client.0.smithi145.stdout:1: exclusive lock released 2024-03-07T03:29:26.740 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ExclusiveLock (528 ms) 2024-03-07T03:29:26.740 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.BreakLock 2024-03-07T03:29:26.760 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:27.271 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.BreakLock (532 ms) 2024-03-07T03:29:27.272 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.DiscardAfterWrite 2024-03-07T03:29:27.272 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:27.361 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.DiscardAfterWrite (89 ms) 2024-03-07T03:29:27.362 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.DefaultFeatures 2024-03-07T03:29:27.362 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.DefaultFeatures (1 ms) 2024-03-07T03:29:27.362 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurge 2024-03-07T03:29:27.362 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:27.555 INFO:tasks.workunit.client.0.smithi145.stdout:0.00% done 2024-03-07T03:29:27.556 INFO:tasks.workunit.client.0.smithi145.stdout:1.56% done 2024-03-07T03:29:27.556 INFO:tasks.workunit.client.0.smithi145.stdout:3.12% done 2024-03-07T03:29:27.556 INFO:tasks.workunit.client.0.smithi145.stdout:4.69% done 2024-03-07T03:29:27.556 INFO:tasks.workunit.client.0.smithi145.stdout:6.25% done 2024-03-07T03:29:27.556 INFO:tasks.workunit.client.0.smithi145.stdout:7.81% done 2024-03-07T03:29:27.556 INFO:tasks.workunit.client.0.smithi145.stdout:9.38% done 2024-03-07T03:29:27.556 INFO:tasks.workunit.client.0.smithi145.stdout:10.94% done 2024-03-07T03:29:27.556 INFO:tasks.workunit.client.0.smithi145.stdout:12.50% done 2024-03-07T03:29:27.556 INFO:tasks.workunit.client.0.smithi145.stdout:14.06% done 2024-03-07T03:29:27.556 INFO:tasks.workunit.client.0.smithi145.stdout:15.62% done 2024-03-07T03:29:27.556 INFO:tasks.workunit.client.0.smithi145.stdout:17.19% done 2024-03-07T03:29:27.556 INFO:tasks.workunit.client.0.smithi145.stdout:18.75% done 2024-03-07T03:29:27.556 INFO:tasks.workunit.client.0.smithi145.stdout:20.31% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:21.88% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:23.44% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:25.00% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:26.56% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:28.12% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:29.69% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:31.25% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:32.81% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:34.38% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:35.94% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:37.50% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:39.06% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:40.62% done 2024-03-07T03:29:27.557 INFO:tasks.workunit.client.0.smithi145.stdout:42.19% done 2024-03-07T03:29:27.558 INFO:tasks.workunit.client.0.smithi145.stdout:43.75% done 2024-03-07T03:29:27.558 INFO:tasks.workunit.client.0.smithi145.stdout:45.31% done 2024-03-07T03:29:27.558 INFO:tasks.workunit.client.0.smithi145.stdout:46.88% done 2024-03-07T03:29:27.558 INFO:tasks.workunit.client.0.smithi145.stdout:48.44% done 2024-03-07T03:29:27.558 INFO:tasks.workunit.client.0.smithi145.stdout:50.00% done 2024-03-07T03:29:27.558 INFO:tasks.workunit.client.0.smithi145.stdout:51.56% done 2024-03-07T03:29:27.558 INFO:tasks.workunit.client.0.smithi145.stdout:53.12% done 2024-03-07T03:29:27.558 INFO:tasks.workunit.client.0.smithi145.stdout:54.69% done 2024-03-07T03:29:27.558 INFO:tasks.workunit.client.0.smithi145.stdout:56.25% done 2024-03-07T03:29:27.558 INFO:tasks.workunit.client.0.smithi145.stdout:57.81% done 2024-03-07T03:29:27.558 INFO:tasks.workunit.client.0.smithi145.stdout:59.38% done 2024-03-07T03:29:27.558 INFO:tasks.workunit.client.0.smithi145.stdout:60.94% done 2024-03-07T03:29:27.558 INFO:tasks.workunit.client.0.smithi145.stdout:62.50% done 2024-03-07T03:29:27.559 INFO:tasks.workunit.client.0.smithi145.stdout:64.06% done 2024-03-07T03:29:27.559 INFO:tasks.workunit.client.0.smithi145.stdout:65.62% done 2024-03-07T03:29:27.559 INFO:tasks.workunit.client.0.smithi145.stdout:67.19% done 2024-03-07T03:29:27.559 INFO:tasks.workunit.client.0.smithi145.stdout:68.75% done 2024-03-07T03:29:27.559 INFO:tasks.workunit.client.0.smithi145.stdout:70.31% done 2024-03-07T03:29:27.559 INFO:tasks.workunit.client.0.smithi145.stdout:71.88% done 2024-03-07T03:29:27.559 INFO:tasks.workunit.client.0.smithi145.stdout:73.44% done 2024-03-07T03:29:27.559 INFO:tasks.workunit.client.0.smithi145.stdout:75.00% done 2024-03-07T03:29:27.559 INFO:tasks.workunit.client.0.smithi145.stdout:76.56% done 2024-03-07T03:29:27.559 INFO:tasks.workunit.client.0.smithi145.stdout:78.12% done 2024-03-07T03:29:27.559 INFO:tasks.workunit.client.0.smithi145.stdout:79.69% done 2024-03-07T03:29:27.559 INFO:tasks.workunit.client.0.smithi145.stdout:81.25% done 2024-03-07T03:29:27.559 INFO:tasks.workunit.client.0.smithi145.stdout:82.81% done 2024-03-07T03:29:27.560 INFO:tasks.workunit.client.0.smithi145.stdout:84.38% done 2024-03-07T03:29:27.560 INFO:tasks.workunit.client.0.smithi145.stdout:85.94% done 2024-03-07T03:29:27.560 INFO:tasks.workunit.client.0.smithi145.stdout:87.50% done 2024-03-07T03:29:27.560 INFO:tasks.workunit.client.0.smithi145.stdout:89.06% done 2024-03-07T03:29:27.560 INFO:tasks.workunit.client.0.smithi145.stdout:90.62% done 2024-03-07T03:29:27.560 INFO:tasks.workunit.client.0.smithi145.stdout:92.19% done 2024-03-07T03:29:27.560 INFO:tasks.workunit.client.0.smithi145.stdout:93.75% done 2024-03-07T03:29:27.560 INFO:tasks.workunit.client.0.smithi145.stdout:95.31% done 2024-03-07T03:29:27.560 INFO:tasks.workunit.client.0.smithi145.stdout:96.88% done 2024-03-07T03:29:27.560 INFO:tasks.workunit.client.0.smithi145.stdout:98.44% done 2024-03-07T03:29:27.560 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurge (194 ms) 2024-03-07T03:29:27.560 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay 2024-03-07T03:29:27.560 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:27.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:27 smithi145 bash[82329]: cluster 2024-03-07T03:29:26.178700+0000 mon.a (mon.0) 1311 : cluster 0 osdmap e768: 8 total, 8 up, 8 in 2024-03-07T03:29:27.561 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:27 smithi145 bash[82329]: audit 2024-03-07T03:29:26.868127+0000 mon.a (mon.0) 1312 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/1659536498"}]: dispatch 2024-03-07T03:29:27.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:27 smithi145 bash[79878]: cluster 2024-03-07T03:29:26.178700+0000 mon.a (mon.0) 1311 : cluster 0 osdmap e768: 8 total, 8 up, 8 in 2024-03-07T03:29:27.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:27 smithi145 bash[79878]: audit 2024-03-07T03:29:26.868127+0000 mon.a (mon.0) 1312 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/1659536498"}]: dispatch 2024-03-07T03:29:27.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:27 smithi189 bash[66969]: cluster 2024-03-07T03:29:26.178700+0000 mon.a (mon.0) 1311 : cluster 0 osdmap e768: 8 total, 8 up, 8 in 2024-03-07T03:29:27.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:27 smithi189 bash[66969]: audit 2024-03-07T03:29:26.868127+0000 mon.a (mon.0) 1312 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/1659536498"}]: dispatch 2024-03-07T03:29:27.698 INFO:tasks.workunit.client.0.smithi145.stdout:0.00% done 2024-03-07T03:29:27.698 INFO:tasks.workunit.client.0.smithi145.stdout:1.56% done 2024-03-07T03:29:27.698 INFO:tasks.workunit.client.0.smithi145.stdout:3.12% done 2024-03-07T03:29:27.699 INFO:tasks.workunit.client.0.smithi145.stdout:4.69% done 2024-03-07T03:29:27.699 INFO:tasks.workunit.client.0.smithi145.stdout:6.25% done 2024-03-07T03:29:27.699 INFO:tasks.workunit.client.0.smithi145.stdout:7.81% done 2024-03-07T03:29:27.699 INFO:tasks.workunit.client.0.smithi145.stdout:9.38% done 2024-03-07T03:29:27.699 INFO:tasks.workunit.client.0.smithi145.stdout:10.94% done 2024-03-07T03:29:27.699 INFO:tasks.workunit.client.0.smithi145.stdout:12.50% done 2024-03-07T03:29:27.699 INFO:tasks.workunit.client.0.smithi145.stdout:14.06% done 2024-03-07T03:29:27.699 INFO:tasks.workunit.client.0.smithi145.stdout:15.62% done 2024-03-07T03:29:27.699 INFO:tasks.workunit.client.0.smithi145.stdout:17.19% done 2024-03-07T03:29:27.699 INFO:tasks.workunit.client.0.smithi145.stdout:18.75% done 2024-03-07T03:29:27.699 INFO:tasks.workunit.client.0.smithi145.stdout:20.31% done 2024-03-07T03:29:27.699 INFO:tasks.workunit.client.0.smithi145.stdout:21.88% done 2024-03-07T03:29:27.699 INFO:tasks.workunit.client.0.smithi145.stdout:23.44% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:25.00% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:26.56% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:28.12% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:29.69% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:31.25% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:32.81% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:34.38% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:35.94% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:37.50% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:39.06% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:40.62% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:42.19% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:43.75% done 2024-03-07T03:29:27.700 INFO:tasks.workunit.client.0.smithi145.stdout:45.31% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:46.88% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:48.44% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:50.00% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:51.56% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:53.12% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:54.69% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:56.25% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:57.81% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:59.38% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:60.94% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:62.50% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:64.06% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:65.62% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:67.19% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:68.75% done 2024-03-07T03:29:27.701 INFO:tasks.workunit.client.0.smithi145.stdout:70.31% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:71.88% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:73.44% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:75.00% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:76.56% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:78.12% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:79.69% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:81.25% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:82.81% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:84.38% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:85.94% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:87.50% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:89.06% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:90.62% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:92.19% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:93.75% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:95.31% done 2024-03-07T03:29:27.702 INFO:tasks.workunit.client.0.smithi145.stdout:96.88% done 2024-03-07T03:29:27.703 INFO:tasks.workunit.client.0.smithi145.stdout:98.44% done 2024-03-07T03:29:27.703 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay (142 ms) 2024-03-07T03:29:27.703 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestTrashPurge 2024-03-07T03:29:27.703 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:27.715 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:28.023 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestTrashPurge (325 ms) 2024-03-07T03:29:28.023 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndRestore 2024-03-07T03:29:28.023 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:28.147 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestTrashMoveAndRestore (124 ms) 2024-03-07T03:29:28.147 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestListWatchers 2024-03-07T03:29:28.147 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:28.227 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestListWatchers (80 ms) 2024-03-07T03:29:28.227 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestSetSnapById 2024-03-07T03:29:28.227 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:28.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:28 smithi145 bash[79878]: cluster 2024-03-07T03:29:27.101674+0000 mgr.y (mgr.34107) 528 : cluster 0 pgmap v921: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 77 KiB/s wr, 268 op/s 2024-03-07T03:29:28.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:28 smithi145 bash[79878]: audit 2024-03-07T03:29:27.193683+0000 mon.a (mon.0) 1313 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/1659536498"}]': finished 2024-03-07T03:29:28.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:28 smithi145 bash[79878]: cluster 2024-03-07T03:29:27.202564+0000 mon.a (mon.0) 1314 : cluster 0 osdmap e769: 8 total, 8 up, 8 in 2024-03-07T03:29:28.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:28 smithi145 bash[79878]: cluster 2024-03-07T03:29:27.512342+0000 mon.a (mon.0) 1315 : cluster 0 osdmap e770: 8 total, 8 up, 8 in 2024-03-07T03:29:28.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:28 smithi145 bash[82329]: cluster 2024-03-07T03:29:27.101674+0000 mgr.y (mgr.34107) 528 : cluster 0 pgmap v921: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 77 KiB/s wr, 268 op/s 2024-03-07T03:29:28.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:28 smithi145 bash[82329]: audit 2024-03-07T03:29:27.193683+0000 mon.a (mon.0) 1313 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/1659536498"}]': finished 2024-03-07T03:29:28.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:28 smithi145 bash[82329]: cluster 2024-03-07T03:29:27.202564+0000 mon.a (mon.0) 1314 : cluster 0 osdmap e769: 8 total, 8 up, 8 in 2024-03-07T03:29:28.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:28 smithi145 bash[82329]: cluster 2024-03-07T03:29:27.512342+0000 mon.a (mon.0) 1315 : cluster 0 osdmap e770: 8 total, 8 up, 8 in 2024-03-07T03:29:28.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:28 smithi189 bash[66969]: cluster 2024-03-07T03:29:27.101674+0000 mgr.y (mgr.34107) 528 : cluster 0 pgmap v921: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 77 KiB/s wr, 268 op/s 2024-03-07T03:29:28.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:28 smithi189 bash[66969]: audit 2024-03-07T03:29:27.193683+0000 mon.a (mon.0) 1313 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.145:0/1659536498"}]': finished 2024-03-07T03:29:28.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:28 smithi189 bash[66969]: cluster 2024-03-07T03:29:27.202564+0000 mon.a (mon.0) 1314 : cluster 0 osdmap e769: 8 total, 8 up, 8 in 2024-03-07T03:29:28.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:28 smithi189 bash[66969]: cluster 2024-03-07T03:29:27.512342+0000 mon.a (mon.0) 1315 : cluster 0 osdmap e770: 8 total, 8 up, 8 in 2024-03-07T03:29:29.262 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestSetSnapById (1035 ms) 2024-03-07T03:29:29.262 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.Namespaces 2024-03-07T03:29:29.301 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.Namespaces (40 ms) 2024-03-07T03:29:29.301 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.NamespacesPP 2024-03-07T03:29:29.347 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:29.354 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:29.460 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.NamespacesPP (158 ms) 2024-03-07T03:29:29.460 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.Migration 2024-03-07T03:29:29.460 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:29.460 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:30.001 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.Migration (541 ms) 2024-03-07T03:29:30.001 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.MigrationPP 2024-03-07T03:29:30.001 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:30.001 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:30.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:29 smithi145 bash[79878]: audit 2024-03-07T03:29:28.973747+0000 mon.a (mon.0) 1316 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:30.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:29 smithi145 bash[79878]: audit 2024-03-07T03:29:28.987356+0000 mon.a (mon.0) 1317 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:30.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:29 smithi145 bash[79878]: cluster 2024-03-07T03:29:29.104422+0000 mgr.y (mgr.34107) 529 : cluster 0 pgmap v924: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 241 KiB/s rd, 144 KiB/s wr, 238 op/s 2024-03-07T03:29:30.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:29 smithi145 bash[79878]: cluster 2024-03-07T03:29:29.206439+0000 mon.a (mon.0) 1318 : cluster 0 osdmap e771: 8 total, 8 up, 8 in 2024-03-07T03:29:30.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:29 smithi145 bash[79878]: audit 2024-03-07T03:29:29.708279+0000 mon.a (mon.0) 1319 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:30.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:29 smithi145 bash[79878]: audit 2024-03-07T03:29:29.714181+0000 mon.a (mon.0) 1320 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:30.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:29 smithi145 bash[82329]: audit 2024-03-07T03:29:28.973747+0000 mon.a (mon.0) 1316 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:30.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:29 smithi145 bash[82329]: audit 2024-03-07T03:29:28.987356+0000 mon.a (mon.0) 1317 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:30.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:29 smithi145 bash[82329]: cluster 2024-03-07T03:29:29.104422+0000 mgr.y (mgr.34107) 529 : cluster 0 pgmap v924: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 241 KiB/s rd, 144 KiB/s wr, 238 op/s 2024-03-07T03:29:30.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:29 smithi145 bash[82329]: cluster 2024-03-07T03:29:29.206439+0000 mon.a (mon.0) 1318 : cluster 0 osdmap e771: 8 total, 8 up, 8 in 2024-03-07T03:29:30.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:29 smithi145 bash[82329]: audit 2024-03-07T03:29:29.708279+0000 mon.a (mon.0) 1319 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:30.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:29 smithi145 bash[82329]: audit 2024-03-07T03:29:29.714181+0000 mon.a (mon.0) 1320 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:30.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:29 smithi189 bash[66969]: audit 2024-03-07T03:29:28.973747+0000 mon.a (mon.0) 1316 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:30.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:29 smithi189 bash[66969]: audit 2024-03-07T03:29:28.987356+0000 mon.a (mon.0) 1317 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:30.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:29 smithi189 bash[66969]: cluster 2024-03-07T03:29:29.104422+0000 mgr.y (mgr.34107) 529 : cluster 0 pgmap v924: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 241 KiB/s rd, 144 KiB/s wr, 238 op/s 2024-03-07T03:29:30.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:29 smithi189 bash[66969]: cluster 2024-03-07T03:29:29.206439+0000 mon.a (mon.0) 1318 : cluster 0 osdmap e771: 8 total, 8 up, 8 in 2024-03-07T03:29:30.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:29 smithi189 bash[66969]: audit 2024-03-07T03:29:29.708279+0000 mon.a (mon.0) 1319 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:30.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:29 smithi189 bash[66969]: audit 2024-03-07T03:29:29.714181+0000 mon.a (mon.0) 1320 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:30.858 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.MigrationPP (858 ms) 2024-03-07T03:29:30.859 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestGetAccessTimestamp 2024-03-07T03:29:30.859 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:30.903 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestGetAccessTimestamp (44 ms) 2024-03-07T03:29:30.903 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.TestGetModifyTimestamp 2024-03-07T03:29:30.903 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:30.950 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.TestGetModifyTimestamp (47 ms) 2024-03-07T03:29:30.950 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ZeroOverlapFlatten 2024-03-07T03:29:30.950 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:31.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:30 smithi145 bash[82329]: audit 2024-03-07T03:29:30.336364+0000 mon.a (mon.0) 1321 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:29:31.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:30 smithi145 bash[82329]: audit 2024-03-07T03:29:30.337870+0000 mon.a (mon.0) 1322 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:29:31.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:30 smithi145 bash[82329]: audit 2024-03-07T03:29:30.346127+0000 mon.a (mon.0) 1323 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:31.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:30 smithi145 bash[79878]: audit 2024-03-07T03:29:30.336364+0000 mon.a (mon.0) 1321 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:29:31.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:30 smithi145 bash[79878]: audit 2024-03-07T03:29:30.337870+0000 mon.a (mon.0) 1322 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:29:31.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:30 smithi145 bash[79878]: audit 2024-03-07T03:29:30.346127+0000 mon.a (mon.0) 1323 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:31.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:30 smithi189 bash[66969]: audit 2024-03-07T03:29:30.336364+0000 mon.a (mon.0) 1321 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:29:31.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:30 smithi189 bash[66969]: audit 2024-03-07T03:29:30.337870+0000 mon.a (mon.0) 1322 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:29:31.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:30 smithi189 bash[66969]: audit 2024-03-07T03:29:30.346127+0000 mon.a (mon.0) 1323 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:32.251 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ZeroOverlapFlatten (1301 ms) 2024-03-07T03:29:32.251 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.PoolMetadata 2024-03-07T03:29:32.304 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:31 smithi189 bash[66969]: cluster 2024-03-07T03:29:31.107460+0000 mgr.y (mgr.34107) 530 : cluster 0 pgmap v926: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 357 KiB/s rd, 73 KiB/s wr, 427 op/s 2024-03-07T03:29:32.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:32 smithi145 bash[79878]: cluster 2024-03-07T03:29:31.107460+0000 mgr.y (mgr.34107) 530 : cluster 0 pgmap v926: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 357 KiB/s rd, 73 KiB/s wr, 427 op/s 2024-03-07T03:29:32.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:32 smithi145 bash[82329]: cluster 2024-03-07T03:29:31.107460+0000 mgr.y (mgr.34107) 530 : cluster 0 pgmap v926: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 357 KiB/s rd, 73 KiB/s wr, 427 op/s 2024-03-07T03:29:32.322 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.PoolMetadata (71 ms) 2024-03-07T03:29:32.322 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.PoolMetadataPP 2024-03-07T03:29:32.356 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.PoolMetadataPP (34 ms) 2024-03-07T03:29:32.356 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.Config 2024-03-07T03:29:32.362 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:32.415 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.Config (59 ms) 2024-03-07T03:29:32.415 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ConfigPP 2024-03-07T03:29:32.421 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:32.504 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ConfigPP (89 ms) 2024-03-07T03:29:32.504 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.PoolStatsPP 2024-03-07T03:29:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:29:32 smithi189 bash[59814]: ts=2024-03-07T03:29:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:29:32.753 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:29:32 smithi145 bash[72238]: [07/Mar/2024:03:29:32] ENGINE Bus STOPPING 2024-03-07T03:29:32.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:29:32 smithi145 bash[72238]: [07/Mar/2024:03:29:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:29:32.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:29:32 smithi145 bash[72238]: [07/Mar/2024:03:29:32] ENGINE Bus STOPPED 2024-03-07T03:29:32.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:29:32 smithi145 bash[72238]: [07/Mar/2024:03:29:32] ENGINE Bus STARTING 2024-03-07T03:29:32.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:29:32 smithi145 bash[72238]: [07/Mar/2024:03:29:32] ENGINE Serving on http://:::9283 2024-03-07T03:29:32.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:29:32 smithi145 bash[72238]: [07/Mar/2024:03:29:32] ENGINE Bus STARTED 2024-03-07T03:29:32.754 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:29:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:29:32] "GET /metrics HTTP/1.1" 200 75027 "" "Prometheus/2.43.0" 2024-03-07T03:29:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[79878]: cluster 2024-03-07T03:29:32.006510+0000 mon.a (mon.0) 1324 : cluster 0 osdmap e772: 8 total, 8 up, 8 in 2024-03-07T03:29:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[79878]: audit 2024-03-07T03:29:32.296595+0000 mon.a (mon.0) 1325 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' 2024-03-07T03:29:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[79878]: audit 2024-03-07T03:29:32.342726+0000 mon.a (mon.0) 1326 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:29:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[79878]: audit 2024-03-07T03:29:32.345080+0000 mon.a (mon.0) 1327 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:29:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[79878]: audit 2024-03-07T03:29:32.346130+0000 mon.a (mon.0) 1328 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:29:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[79878]: audit 2024-03-07T03:29:32.352236+0000 mon.a (mon.0) 1329 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[79878]: cluster 2024-03-07T03:29:32.506719+0000 mon.a (mon.0) 1330 : cluster 0 osdmap e773: 8 total, 8 up, 8 in 2024-03-07T03:29:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[79878]: audit 2024-03-07T03:29:32.971673+0000 mon.a (mon.0) 1331 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[82329]: cluster 2024-03-07T03:29:32.006510+0000 mon.a (mon.0) 1324 : cluster 0 osdmap e772: 8 total, 8 up, 8 in 2024-03-07T03:29:33.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[82329]: audit 2024-03-07T03:29:32.296595+0000 mon.a (mon.0) 1325 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' 2024-03-07T03:29:33.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[82329]: audit 2024-03-07T03:29:32.342726+0000 mon.a (mon.0) 1326 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:29:33.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[82329]: audit 2024-03-07T03:29:32.345080+0000 mon.a (mon.0) 1327 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:29:33.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[82329]: audit 2024-03-07T03:29:32.346130+0000 mon.a (mon.0) 1328 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:29:33.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[82329]: audit 2024-03-07T03:29:32.352236+0000 mon.a (mon.0) 1329 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:33.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[82329]: cluster 2024-03-07T03:29:32.506719+0000 mon.a (mon.0) 1330 : cluster 0 osdmap e773: 8 total, 8 up, 8 in 2024-03-07T03:29:33.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:33 smithi145 bash[82329]: audit 2024-03-07T03:29:32.971673+0000 mon.a (mon.0) 1331 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:33 smithi189 bash[66969]: cluster 2024-03-07T03:29:32.006510+0000 mon.a (mon.0) 1324 : cluster 0 osdmap e772: 8 total, 8 up, 8 in 2024-03-07T03:29:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:33 smithi189 bash[66969]: audit 2024-03-07T03:29:32.296595+0000 mon.a (mon.0) 1325 : audit 1 from='client.? 172.21.15.145:0/280138442' entity='client.admin' 2024-03-07T03:29:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:33 smithi189 bash[66969]: audit 2024-03-07T03:29:32.342726+0000 mon.a (mon.0) 1326 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:29:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:33 smithi189 bash[66969]: audit 2024-03-07T03:29:32.345080+0000 mon.a (mon.0) 1327 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:29:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:33 smithi189 bash[66969]: audit 2024-03-07T03:29:32.346130+0000 mon.a (mon.0) 1328 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:29:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:33 smithi189 bash[66969]: audit 2024-03-07T03:29:32.352236+0000 mon.a (mon.0) 1329 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:33 smithi189 bash[66969]: cluster 2024-03-07T03:29:32.506719+0000 mon.a (mon.0) 1330 : cluster 0 osdmap e773: 8 total, 8 up, 8 in 2024-03-07T03:29:33.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:33 smithi189 bash[66969]: audit 2024-03-07T03:29:32.971673+0000 mon.a (mon.0) 1331 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:34.523 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:34.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:34 smithi145 bash[79878]: cluster 2024-03-07T03:29:33.108479+0000 mgr.y (mgr.34107) 531 : cluster 0 pgmap v929: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 404 KiB/s rd, 68 KiB/s wr, 489 op/s 2024-03-07T03:29:34.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:34 smithi145 bash[79878]: cluster 2024-03-07T03:29:33.514263+0000 mon.a (mon.0) 1332 : cluster 0 osdmap e774: 8 total, 8 up, 8 in 2024-03-07T03:29:34.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:34 smithi145 bash[79878]: audit 2024-03-07T03:29:33.518147+0000 mon.a (mon.0) 1333 : audit 1 from='client.? 172.21.15.145:0/2963311994' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:34 smithi145 bash[82329]: cluster 2024-03-07T03:29:33.108479+0000 mgr.y (mgr.34107) 531 : cluster 0 pgmap v929: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 404 KiB/s rd, 68 KiB/s wr, 489 op/s 2024-03-07T03:29:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:34 smithi145 bash[82329]: cluster 2024-03-07T03:29:33.514263+0000 mon.a (mon.0) 1332 : cluster 0 osdmap e774: 8 total, 8 up, 8 in 2024-03-07T03:29:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:34 smithi145 bash[82329]: audit 2024-03-07T03:29:33.518147+0000 mon.a (mon.0) 1333 : audit 1 from='client.? 172.21.15.145:0/2963311994' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:34.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:34 smithi189 bash[66969]: cluster 2024-03-07T03:29:33.108479+0000 mgr.y (mgr.34107) 531 : cluster 0 pgmap v929: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 404 KiB/s rd, 68 KiB/s wr, 489 op/s 2024-03-07T03:29:34.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:34 smithi189 bash[66969]: cluster 2024-03-07T03:29:33.514263+0000 mon.a (mon.0) 1332 : cluster 0 osdmap e774: 8 total, 8 up, 8 in 2024-03-07T03:29:34.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:34 smithi189 bash[66969]: audit 2024-03-07T03:29:33.518147+0000 mon.a (mon.0) 1333 : audit 1 from='client.? 172.21.15.145:0/2963311994' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:35.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:35 smithi145 bash[79878]: audit 2024-03-07T03:29:34.511076+0000 mon.a (mon.0) 1334 : audit 1 from='client.? 172.21.15.145:0/2963311994' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:35.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:35 smithi145 bash[79878]: cluster 2024-03-07T03:29:34.523655+0000 mon.a (mon.0) 1335 : cluster 0 osdmap e775: 8 total, 8 up, 8 in 2024-03-07T03:29:35.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:35 smithi145 bash[82329]: audit 2024-03-07T03:29:34.511076+0000 mon.a (mon.0) 1334 : audit 1 from='client.? 172.21.15.145:0/2963311994' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:35.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:35 smithi145 bash[82329]: cluster 2024-03-07T03:29:34.523655+0000 mon.a (mon.0) 1335 : cluster 0 osdmap e775: 8 total, 8 up, 8 in 2024-03-07T03:29:35.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:35 smithi189 bash[66969]: audit 2024-03-07T03:29:34.511076+0000 mon.a (mon.0) 1334 : audit 1 from='client.? 172.21.15.145:0/2963311994' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:35.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:35 smithi189 bash[66969]: cluster 2024-03-07T03:29:34.523655+0000 mon.a (mon.0) 1335 : cluster 0 osdmap e775: 8 total, 8 up, 8 in 2024-03-07T03:29:36.654 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:36.669 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:36.685 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:36.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:36 smithi145 bash[79878]: cluster 2024-03-07T03:29:35.110366+0000 mgr.y (mgr.34107) 532 : cluster 0 pgmap v932: 649 pgs: 7 creating+peering, 20 unknown, 622 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 502 KiB/s rd, 31 KiB/s wr, 653 op/s 2024-03-07T03:29:36.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:36 smithi145 bash[79878]: cluster 2024-03-07T03:29:35.544856+0000 mon.a (mon.0) 1336 : cluster 0 osdmap e776: 8 total, 8 up, 8 in 2024-03-07T03:29:36.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:36 smithi145 bash[82329]: cluster 2024-03-07T03:29:35.110366+0000 mgr.y (mgr.34107) 532 : cluster 0 pgmap v932: 649 pgs: 7 creating+peering, 20 unknown, 622 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 502 KiB/s rd, 31 KiB/s wr, 653 op/s 2024-03-07T03:29:36.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:36 smithi145 bash[82329]: cluster 2024-03-07T03:29:35.544856+0000 mon.a (mon.0) 1336 : cluster 0 osdmap e776: 8 total, 8 up, 8 in 2024-03-07T03:29:36.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:36 smithi189 bash[66969]: cluster 2024-03-07T03:29:35.110366+0000 mgr.y (mgr.34107) 532 : cluster 0 pgmap v932: 649 pgs: 7 creating+peering, 20 unknown, 622 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 502 KiB/s rd, 31 KiB/s wr, 653 op/s 2024-03-07T03:29:36.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:36 smithi189 bash[66969]: cluster 2024-03-07T03:29:35.544856+0000 mon.a (mon.0) 1336 : cluster 0 osdmap e776: 8 total, 8 up, 8 in 2024-03-07T03:29:37.762 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.PoolStatsPP (5258 ms) 2024-03-07T03:29:37.762 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ImageSpec 2024-03-07T03:29:37.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:37 smithi145 bash[79878]: cluster 2024-03-07T03:29:36.548413+0000 mon.a (mon.0) 1337 : cluster 0 osdmap e777: 8 total, 8 up, 8 in 2024-03-07T03:29:37.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:37 smithi145 bash[82329]: cluster 2024-03-07T03:29:36.548413+0000 mon.a (mon.0) 1337 : cluster 0 osdmap e777: 8 total, 8 up, 8 in 2024-03-07T03:29:37.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:37 smithi189 bash[66969]: cluster 2024-03-07T03:29:36.548413+0000 mon.a (mon.0) 1337 : cluster 0 osdmap e777: 8 total, 8 up, 8 in 2024-03-07T03:29:38.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:38 smithi145 bash[79878]: cluster 2024-03-07T03:29:37.112808+0000 mgr.y (mgr.34107) 533 : cluster 0 pgmap v935: 649 pgs: 7 creating+peering, 642 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 398 KiB/s rd, 30 KiB/s wr, 531 op/s 2024-03-07T03:29:38.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:38 smithi145 bash[79878]: cluster 2024-03-07T03:29:37.569079+0000 mon.a (mon.0) 1338 : cluster 0 osdmap e778: 8 total, 8 up, 8 in 2024-03-07T03:29:38.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:38 smithi145 bash[82329]: cluster 2024-03-07T03:29:37.112808+0000 mgr.y (mgr.34107) 533 : cluster 0 pgmap v935: 649 pgs: 7 creating+peering, 642 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 398 KiB/s rd, 30 KiB/s wr, 531 op/s 2024-03-07T03:29:38.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:38 smithi145 bash[82329]: cluster 2024-03-07T03:29:37.569079+0000 mon.a (mon.0) 1338 : cluster 0 osdmap e778: 8 total, 8 up, 8 in 2024-03-07T03:29:38.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:38 smithi189 bash[66969]: cluster 2024-03-07T03:29:37.112808+0000 mgr.y (mgr.34107) 533 : cluster 0 pgmap v935: 649 pgs: 7 creating+peering, 642 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 398 KiB/s rd, 30 KiB/s wr, 531 op/s 2024-03-07T03:29:38.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:38 smithi189 bash[66969]: cluster 2024-03-07T03:29:37.569079+0000 mon.a (mon.0) 1338 : cluster 0 osdmap e778: 8 total, 8 up, 8 in 2024-03-07T03:29:39.592 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:40.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:39 smithi145 bash[79878]: cluster 2024-03-07T03:29:38.582802+0000 mon.a (mon.0) 1339 : cluster 0 osdmap e779: 8 total, 8 up, 8 in 2024-03-07T03:29:40.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:39 smithi145 bash[79878]: audit 2024-03-07T03:29:38.597167+0000 mon.a (mon.0) 1340 : audit 1 from='client.? 172.21.15.145:0/1831629798' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:40.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:39 smithi145 bash[82329]: cluster 2024-03-07T03:29:38.582802+0000 mon.a (mon.0) 1339 : cluster 0 osdmap e779: 8 total, 8 up, 8 in 2024-03-07T03:29:40.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:39 smithi145 bash[82329]: audit 2024-03-07T03:29:38.597167+0000 mon.a (mon.0) 1340 : audit 1 from='client.? 172.21.15.145:0/1831629798' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:39 smithi189 bash[66969]: cluster 2024-03-07T03:29:38.582802+0000 mon.a (mon.0) 1339 : cluster 0 osdmap e779: 8 total, 8 up, 8 in 2024-03-07T03:29:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:39 smithi189 bash[66969]: audit 2024-03-07T03:29:38.597167+0000 mon.a (mon.0) 1340 : audit 1 from='client.? 172.21.15.145:0/1831629798' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:40.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:40 smithi189 bash[66969]: cluster 2024-03-07T03:29:39.115787+0000 mgr.y (mgr.34107) 534 : cluster 0 pgmap v938: 681 pgs: 10 creating+peering, 22 unknown, 649 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 8.0 KiB/s wr, 79 op/s 2024-03-07T03:29:40.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:40 smithi189 bash[66969]: cluster 2024-03-07T03:29:39.574901+0000 mon.a (mon.0) 1341 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:29:40.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:40 smithi189 bash[66969]: audit 2024-03-07T03:29:39.588439+0000 mon.a (mon.0) 1342 : audit 1 from='client.? 172.21.15.145:0/1831629798' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:40.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:40 smithi189 bash[66969]: cluster 2024-03-07T03:29:39.597977+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e780: 8 total, 8 up, 8 in 2024-03-07T03:29:41.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:40 smithi145 bash[79878]: cluster 2024-03-07T03:29:39.115787+0000 mgr.y (mgr.34107) 534 : cluster 0 pgmap v938: 681 pgs: 10 creating+peering, 22 unknown, 649 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 8.0 KiB/s wr, 79 op/s 2024-03-07T03:29:41.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:40 smithi145 bash[79878]: cluster 2024-03-07T03:29:39.574901+0000 mon.a (mon.0) 1341 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:29:41.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:40 smithi145 bash[79878]: audit 2024-03-07T03:29:39.588439+0000 mon.a (mon.0) 1342 : audit 1 from='client.? 172.21.15.145:0/1831629798' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:41.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:40 smithi145 bash[79878]: cluster 2024-03-07T03:29:39.597977+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e780: 8 total, 8 up, 8 in 2024-03-07T03:29:41.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:40 smithi145 bash[82329]: cluster 2024-03-07T03:29:39.115787+0000 mgr.y (mgr.34107) 534 : cluster 0 pgmap v938: 681 pgs: 10 creating+peering, 22 unknown, 649 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 8.0 KiB/s wr, 79 op/s 2024-03-07T03:29:41.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:40 smithi145 bash[82329]: cluster 2024-03-07T03:29:39.574901+0000 mon.a (mon.0) 1341 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:29:41.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:40 smithi145 bash[82329]: audit 2024-03-07T03:29:39.588439+0000 mon.a (mon.0) 1342 : audit 1 from='client.? 172.21.15.145:0/1831629798' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:41.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:40 smithi145 bash[82329]: cluster 2024-03-07T03:29:39.597977+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e780: 8 total, 8 up, 8 in 2024-03-07T03:29:42.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:41 smithi145 bash[79878]: cluster 2024-03-07T03:29:40.605489+0000 mon.a (mon.0) 1344 : cluster 0 osdmap e781: 8 total, 8 up, 8 in 2024-03-07T03:29:42.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:41 smithi145 bash[82329]: cluster 2024-03-07T03:29:40.605489+0000 mon.a (mon.0) 1344 : cluster 0 osdmap e781: 8 total, 8 up, 8 in 2024-03-07T03:29:42.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:41 smithi189 bash[66969]: cluster 2024-03-07T03:29:40.605489+0000 mon.a (mon.0) 1344 : cluster 0 osdmap e781: 8 total, 8 up, 8 in 2024-03-07T03:29:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:29:42 smithi189 bash[59814]: ts=2024-03-07T03:29:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:29:43.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:29:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:29:42] "GET /metrics HTTP/1.1" 200 75039 "" "Prometheus/2.43.0" 2024-03-07T03:29:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:42 smithi145 bash[79878]: cluster 2024-03-07T03:29:41.118299+0000 mgr.y (mgr.34107) 535 : cluster 0 pgmap v941: 681 pgs: 10 creating+peering, 11 unknown, 660 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 4.0 KiB/s wr, 26 op/s 2024-03-07T03:29:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:42 smithi145 bash[79878]: cluster 2024-03-07T03:29:41.599094+0000 mon.a (mon.0) 1345 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:29:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:42 smithi145 bash[79878]: cluster 2024-03-07T03:29:41.620713+0000 mon.a (mon.0) 1346 : cluster 0 osdmap e782: 8 total, 8 up, 8 in 2024-03-07T03:29:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:42 smithi145 bash[79878]: cluster 2024-03-07T03:29:42.513763+0000 mon.a (mon.0) 1347 : cluster 0 osdmap e783: 8 total, 8 up, 8 in 2024-03-07T03:29:43.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:42 smithi145 bash[82329]: cluster 2024-03-07T03:29:41.118299+0000 mgr.y (mgr.34107) 535 : cluster 0 pgmap v941: 681 pgs: 10 creating+peering, 11 unknown, 660 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 4.0 KiB/s wr, 26 op/s 2024-03-07T03:29:43.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:42 smithi145 bash[82329]: cluster 2024-03-07T03:29:41.599094+0000 mon.a (mon.0) 1345 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:29:43.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:42 smithi145 bash[82329]: cluster 2024-03-07T03:29:41.620713+0000 mon.a (mon.0) 1346 : cluster 0 osdmap e782: 8 total, 8 up, 8 in 2024-03-07T03:29:43.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:42 smithi145 bash[82329]: cluster 2024-03-07T03:29:42.513763+0000 mon.a (mon.0) 1347 : cluster 0 osdmap e783: 8 total, 8 up, 8 in 2024-03-07T03:29:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:42 smithi189 bash[66969]: cluster 2024-03-07T03:29:41.118299+0000 mgr.y (mgr.34107) 535 : cluster 0 pgmap v941: 681 pgs: 10 creating+peering, 11 unknown, 660 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 4.0 KiB/s wr, 26 op/s 2024-03-07T03:29:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:42 smithi189 bash[66969]: cluster 2024-03-07T03:29:41.599094+0000 mon.a (mon.0) 1345 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:29:43.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:42 smithi189 bash[66969]: cluster 2024-03-07T03:29:41.620713+0000 mon.a (mon.0) 1346 : cluster 0 osdmap e782: 8 total, 8 up, 8 in 2024-03-07T03:29:43.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:42 smithi189 bash[66969]: cluster 2024-03-07T03:29:42.513763+0000 mon.a (mon.0) 1347 : cluster 0 osdmap e783: 8 total, 8 up, 8 in 2024-03-07T03:29:43.999 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ImageSpec (6237 ms) 2024-03-07T03:29:43.999 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.SnapRemoveWithChildMissing 2024-03-07T03:29:45.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:44 smithi145 bash[79878]: cluster 2024-03-07T03:29:43.119596+0000 mgr.y (mgr.34107) 536 : cluster 0 pgmap v944: 681 pgs: 10 creating+peering, 671 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.2 KiB/s wr, 37 op/s 2024-03-07T03:29:45.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:44 smithi145 bash[79878]: cluster 2024-03-07T03:29:43.631398+0000 mon.a (mon.0) 1348 : cluster 0 osdmap e784: 8 total, 8 up, 8 in 2024-03-07T03:29:45.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:44 smithi145 bash[82329]: cluster 2024-03-07T03:29:43.119596+0000 mgr.y (mgr.34107) 536 : cluster 0 pgmap v944: 681 pgs: 10 creating+peering, 671 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.2 KiB/s wr, 37 op/s 2024-03-07T03:29:45.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:44 smithi145 bash[82329]: cluster 2024-03-07T03:29:43.631398+0000 mon.a (mon.0) 1348 : cluster 0 osdmap e784: 8 total, 8 up, 8 in 2024-03-07T03:29:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:44 smithi189 bash[66969]: cluster 2024-03-07T03:29:43.119596+0000 mgr.y (mgr.34107) 536 : cluster 0 pgmap v944: 681 pgs: 10 creating+peering, 671 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.2 KiB/s wr, 37 op/s 2024-03-07T03:29:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:44 smithi189 bash[66969]: cluster 2024-03-07T03:29:43.631398+0000 mon.a (mon.0) 1348 : cluster 0 osdmap e784: 8 total, 8 up, 8 in 2024-03-07T03:29:45.645 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:46.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:45 smithi145 bash[79878]: cluster 2024-03-07T03:29:44.640380+0000 mon.a (mon.0) 1349 : cluster 0 osdmap e785: 8 total, 8 up, 8 in 2024-03-07T03:29:46.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:45 smithi145 bash[79878]: audit 2024-03-07T03:29:44.642637+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.145:0/2222157302' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:46.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:45 smithi145 bash[79878]: audit 2024-03-07T03:29:44.658469+0000 mon.a (mon.0) 1350 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:46.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:45 smithi145 bash[79878]: audit 2024-03-07T03:29:45.633843+0000 mon.a (mon.0) 1351 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:46.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:45 smithi145 bash[79878]: cluster 2024-03-07T03:29:45.643150+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e786: 8 total, 8 up, 8 in 2024-03-07T03:29:46.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:45 smithi145 bash[82329]: cluster 2024-03-07T03:29:44.640380+0000 mon.a (mon.0) 1349 : cluster 0 osdmap e785: 8 total, 8 up, 8 in 2024-03-07T03:29:46.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:45 smithi145 bash[82329]: audit 2024-03-07T03:29:44.642637+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.145:0/2222157302' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:46.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:45 smithi145 bash[82329]: audit 2024-03-07T03:29:44.658469+0000 mon.a (mon.0) 1350 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:46.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:45 smithi145 bash[82329]: audit 2024-03-07T03:29:45.633843+0000 mon.a (mon.0) 1351 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:46.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:45 smithi145 bash[82329]: cluster 2024-03-07T03:29:45.643150+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e786: 8 total, 8 up, 8 in 2024-03-07T03:29:46.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:45 smithi189 bash[66969]: cluster 2024-03-07T03:29:44.640380+0000 mon.a (mon.0) 1349 : cluster 0 osdmap e785: 8 total, 8 up, 8 in 2024-03-07T03:29:46.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:45 smithi189 bash[66969]: audit 2024-03-07T03:29:44.642637+0000 mon.c (mon.2) 45 : audit 1 from='client.? 172.21.15.145:0/2222157302' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:46.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:45 smithi189 bash[66969]: audit 2024-03-07T03:29:44.658469+0000 mon.a (mon.0) 1350 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:46.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:45 smithi189 bash[66969]: audit 2024-03-07T03:29:45.633843+0000 mon.a (mon.0) 1351 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:46.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:45 smithi189 bash[66969]: cluster 2024-03-07T03:29:45.643150+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e786: 8 total, 8 up, 8 in 2024-03-07T03:29:47.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:46 smithi145 bash[79878]: cluster 2024-03-07T03:29:45.121646+0000 mgr.y (mgr.34107) 537 : cluster 0 pgmap v947: 713 pgs: 4 creating+peering, 28 unknown, 681 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 4.2 KiB/s wr, 78 op/s 2024-03-07T03:29:47.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:46 smithi145 bash[79878]: cluster 2024-03-07T03:29:45.645947+0000 mon.a (mon.0) 1353 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:29:47.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:46 smithi145 bash[79878]: cluster 2024-03-07T03:29:46.663431+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e787: 8 total, 8 up, 8 in 2024-03-07T03:29:47.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:46 smithi145 bash[82329]: cluster 2024-03-07T03:29:45.121646+0000 mgr.y (mgr.34107) 537 : cluster 0 pgmap v947: 713 pgs: 4 creating+peering, 28 unknown, 681 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 4.2 KiB/s wr, 78 op/s 2024-03-07T03:29:47.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:46 smithi145 bash[82329]: cluster 2024-03-07T03:29:45.645947+0000 mon.a (mon.0) 1353 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:29:47.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:46 smithi145 bash[82329]: cluster 2024-03-07T03:29:46.663431+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e787: 8 total, 8 up, 8 in 2024-03-07T03:29:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:46 smithi189 bash[66969]: cluster 2024-03-07T03:29:45.121646+0000 mgr.y (mgr.34107) 537 : cluster 0 pgmap v947: 713 pgs: 4 creating+peering, 28 unknown, 681 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 4.2 KiB/s wr, 78 op/s 2024-03-07T03:29:47.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:46 smithi189 bash[66969]: cluster 2024-03-07T03:29:45.645947+0000 mon.a (mon.0) 1353 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:29:47.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:46 smithi189 bash[66969]: cluster 2024-03-07T03:29:46.663431+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e787: 8 total, 8 up, 8 in 2024-03-07T03:29:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:48 smithi145 bash[79878]: cluster 2024-03-07T03:29:47.123665+0000 mgr.y (mgr.34107) 538 : cluster 0 pgmap v950: 713 pgs: 4 creating+peering, 13 unknown, 696 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 5.7 KiB/s wr, 165 op/s 2024-03-07T03:29:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:48 smithi145 bash[79878]: audit 2024-03-07T03:29:47.660815+0000 mon.a (mon.0) 1355 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:48 smithi145 bash[79878]: audit 2024-03-07T03:29:47.664484+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:48 smithi145 bash[79878]: cluster 2024-03-07T03:29:47.685302+0000 mon.a (mon.0) 1357 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:29:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:48 smithi145 bash[79878]: cluster 2024-03-07T03:29:47.707940+0000 mon.a (mon.0) 1358 : cluster 0 osdmap e788: 8 total, 8 up, 8 in 2024-03-07T03:29:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:48 smithi145 bash[82329]: cluster 2024-03-07T03:29:47.123665+0000 mgr.y (mgr.34107) 538 : cluster 0 pgmap v950: 713 pgs: 4 creating+peering, 13 unknown, 696 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 5.7 KiB/s wr, 165 op/s 2024-03-07T03:29:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:48 smithi145 bash[82329]: audit 2024-03-07T03:29:47.660815+0000 mon.a (mon.0) 1355 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:48 smithi145 bash[82329]: audit 2024-03-07T03:29:47.664484+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:48 smithi145 bash[82329]: cluster 2024-03-07T03:29:47.685302+0000 mon.a (mon.0) 1357 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:29:49.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:48 smithi145 bash[82329]: cluster 2024-03-07T03:29:47.707940+0000 mon.a (mon.0) 1358 : cluster 0 osdmap e788: 8 total, 8 up, 8 in 2024-03-07T03:29:49.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:48 smithi189 bash[66969]: cluster 2024-03-07T03:29:47.123665+0000 mgr.y (mgr.34107) 538 : cluster 0 pgmap v950: 713 pgs: 4 creating+peering, 13 unknown, 696 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 5.7 KiB/s wr, 165 op/s 2024-03-07T03:29:49.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:48 smithi189 bash[66969]: audit 2024-03-07T03:29:47.660815+0000 mon.a (mon.0) 1355 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:29:49.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:48 smithi189 bash[66969]: audit 2024-03-07T03:29:47.664484+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:49.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:48 smithi189 bash[66969]: cluster 2024-03-07T03:29:47.685302+0000 mon.a (mon.0) 1357 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:29:49.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:48 smithi189 bash[66969]: cluster 2024-03-07T03:29:47.707940+0000 mon.a (mon.0) 1358 : cluster 0 osdmap e788: 8 total, 8 up, 8 in 2024-03-07T03:29:50.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:49 smithi145 bash[79878]: cluster 2024-03-07T03:29:48.703489+0000 mon.a (mon.0) 1359 : cluster 0 osdmap e789: 8 total, 8 up, 8 in 2024-03-07T03:29:50.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:49 smithi145 bash[82329]: cluster 2024-03-07T03:29:48.703489+0000 mon.a (mon.0) 1359 : cluster 0 osdmap e789: 8 total, 8 up, 8 in 2024-03-07T03:29:50.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:49 smithi189 bash[66969]: cluster 2024-03-07T03:29:48.703489+0000 mon.a (mon.0) 1359 : cluster 0 osdmap e789: 8 total, 8 up, 8 in 2024-03-07T03:29:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:50 smithi145 bash[82329]: cluster 2024-03-07T03:29:49.126599+0000 mgr.y (mgr.34107) 539 : cluster 0 pgmap v953: 713 pgs: 4 creating+peering, 709 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 4.5 KiB/s wr, 125 op/s 2024-03-07T03:29:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:50 smithi145 bash[82329]: cluster 2024-03-07T03:29:49.708067+0000 mon.a (mon.0) 1360 : cluster 0 osdmap e790: 8 total, 8 up, 8 in 2024-03-07T03:29:51.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:50 smithi145 bash[79878]: cluster 2024-03-07T03:29:49.126599+0000 mgr.y (mgr.34107) 539 : cluster 0 pgmap v953: 713 pgs: 4 creating+peering, 709 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 4.5 KiB/s wr, 125 op/s 2024-03-07T03:29:51.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:50 smithi145 bash[79878]: cluster 2024-03-07T03:29:49.708067+0000 mon.a (mon.0) 1360 : cluster 0 osdmap e790: 8 total, 8 up, 8 in 2024-03-07T03:29:51.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:50 smithi189 bash[66969]: cluster 2024-03-07T03:29:49.126599+0000 mgr.y (mgr.34107) 539 : cluster 0 pgmap v953: 713 pgs: 4 creating+peering, 709 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 4.5 KiB/s wr, 125 op/s 2024-03-07T03:29:51.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:50 smithi189 bash[66969]: cluster 2024-03-07T03:29:49.708067+0000 mon.a (mon.0) 1360 : cluster 0 osdmap e790: 8 total, 8 up, 8 in 2024-03-07T03:29:52.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:51 smithi145 bash[79878]: cluster 2024-03-07T03:29:50.736080+0000 mon.a (mon.0) 1361 : cluster 0 osdmap e791: 8 total, 8 up, 8 in 2024-03-07T03:29:52.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:51 smithi145 bash[82329]: cluster 2024-03-07T03:29:50.736080+0000 mon.a (mon.0) 1361 : cluster 0 osdmap e791: 8 total, 8 up, 8 in 2024-03-07T03:29:52.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:51 smithi189 bash[66969]: cluster 2024-03-07T03:29:50.736080+0000 mon.a (mon.0) 1361 : cluster 0 osdmap e791: 8 total, 8 up, 8 in 2024-03-07T03:29:52.570 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 3 2024-03-07T03:29:52.570 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 3 2024-03-07T03:29:52.570 INFO:tasks.workunit.client.0.smithi145.stdout:child: image218 2024-03-07T03:29:52.570 INFO:tasks.workunit.client.0.smithi145.stdout:child: image220 2024-03-07T03:29:52.571 INFO:tasks.workunit.client.0.smithi145.stdout:child: image219 2024-03-07T03:29:52.571 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-15/image218/85fedffaefd4 2024-03-07T03:29:52.571 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:29:52.571 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-34/image219/85fe1a143729 2024-03-07T03:29:52.571 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:29:52.571 INFO:tasks.workunit.client.0.smithi145.stdout:found child test-librbd-smithi145-68337-15/image220/85fe8a3ea40e 2024-03-07T03:29:52.571 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:29:52.571 INFO:tasks.workunit.client.0.smithi145.stdout:num children is: 0 2024-03-07T03:29:52.571 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 0 2024-03-07T03:29:52.571 INFO:tasks.workunit.client.0.smithi145.stdout:num snaps is: 0 2024-03-07T03:29:52.571 INFO:tasks.workunit.client.0.smithi145.stdout:expected: 0 2024-03-07T03:29:52.571 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.SnapRemoveWithChildMissing (8571 ms) 2024-03-07T03:29:52.571 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.QuiesceWatch 2024-03-07T03:29:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:29:52 smithi189 bash[59814]: ts=2024-03-07T03:29:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:29:53.056 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:29:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:29:52] "GET /metrics HTTP/1.1" 200 82210 "" "Prometheus/2.43.0" 2024-03-07T03:29:53.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:53 smithi145 bash[79878]: cluster 2024-03-07T03:29:51.129162+0000 mgr.y (mgr.34107) 540 : cluster 0 pgmap v956: 681 pgs: 681 active+clean; 2.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:29:53.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:53 smithi145 bash[79878]: cluster 2024-03-07T03:29:51.740781+0000 mon.a (mon.0) 1362 : cluster 0 osdmap e792: 8 total, 8 up, 8 in 2024-03-07T03:29:53.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:53 smithi145 bash[79878]: cluster 2024-03-07T03:29:52.517619+0000 mon.a (mon.0) 1363 : cluster 0 osdmap e793: 8 total, 8 up, 8 in 2024-03-07T03:29:53.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:53 smithi145 bash[82329]: cluster 2024-03-07T03:29:51.129162+0000 mgr.y (mgr.34107) 540 : cluster 0 pgmap v956: 681 pgs: 681 active+clean; 2.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:29:53.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:53 smithi145 bash[82329]: cluster 2024-03-07T03:29:51.740781+0000 mon.a (mon.0) 1362 : cluster 0 osdmap e792: 8 total, 8 up, 8 in 2024-03-07T03:29:53.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:53 smithi145 bash[82329]: cluster 2024-03-07T03:29:52.517619+0000 mon.a (mon.0) 1363 : cluster 0 osdmap e793: 8 total, 8 up, 8 in 2024-03-07T03:29:53.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:53 smithi189 bash[66969]: cluster 2024-03-07T03:29:51.129162+0000 mgr.y (mgr.34107) 540 : cluster 0 pgmap v956: 681 pgs: 681 active+clean; 2.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:29:53.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:53 smithi189 bash[66969]: cluster 2024-03-07T03:29:51.740781+0000 mon.a (mon.0) 1362 : cluster 0 osdmap e792: 8 total, 8 up, 8 in 2024-03-07T03:29:53.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:53 smithi189 bash[66969]: cluster 2024-03-07T03:29:52.517619+0000 mon.a (mon.0) 1363 : cluster 0 osdmap e793: 8 total, 8 up, 8 in 2024-03-07T03:29:54.532 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:29:54.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:54 smithi145 bash[79878]: cluster 2024-03-07T03:29:53.130291+0000 mgr.y (mgr.34107) 541 : cluster 0 pgmap v959: 681 pgs: 681 active+clean; 2.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:29:54.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:54 smithi145 bash[79878]: cluster 2024-03-07T03:29:53.520700+0000 mon.a (mon.0) 1364 : cluster 0 osdmap e794: 8 total, 8 up, 8 in 2024-03-07T03:29:54.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:54 smithi145 bash[79878]: audit 2024-03-07T03:29:53.522113+0000 mon.a (mon.0) 1365 : audit 1 from='client.? 172.21.15.145:0/3123636824' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:54 smithi145 bash[82329]: cluster 2024-03-07T03:29:53.130291+0000 mgr.y (mgr.34107) 541 : cluster 0 pgmap v959: 681 pgs: 681 active+clean; 2.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:29:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:54 smithi145 bash[82329]: cluster 2024-03-07T03:29:53.520700+0000 mon.a (mon.0) 1364 : cluster 0 osdmap e794: 8 total, 8 up, 8 in 2024-03-07T03:29:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:54 smithi145 bash[82329]: audit 2024-03-07T03:29:53.522113+0000 mon.a (mon.0) 1365 : audit 1 from='client.? 172.21.15.145:0/3123636824' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:54.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:54 smithi189 bash[66969]: cluster 2024-03-07T03:29:53.130291+0000 mgr.y (mgr.34107) 541 : cluster 0 pgmap v959: 681 pgs: 681 active+clean; 2.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:29:54.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:54 smithi189 bash[66969]: cluster 2024-03-07T03:29:53.520700+0000 mon.a (mon.0) 1364 : cluster 0 osdmap e794: 8 total, 8 up, 8 in 2024-03-07T03:29:54.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:54 smithi189 bash[66969]: audit 2024-03-07T03:29:53.522113+0000 mon.a (mon.0) 1365 : audit 1 from='client.? 172.21.15.145:0/3123636824' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:55.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:55 smithi145 bash[79878]: audit 2024-03-07T03:29:54.517639+0000 mon.a (mon.0) 1366 : audit 1 from='client.? 172.21.15.145:0/3123636824' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:55.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:55 smithi145 bash[79878]: cluster 2024-03-07T03:29:54.527382+0000 mon.a (mon.0) 1367 : cluster 0 osdmap e795: 8 total, 8 up, 8 in 2024-03-07T03:29:55.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:55 smithi145 bash[82329]: audit 2024-03-07T03:29:54.517639+0000 mon.a (mon.0) 1366 : audit 1 from='client.? 172.21.15.145:0/3123636824' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:55.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:55 smithi145 bash[82329]: cluster 2024-03-07T03:29:54.527382+0000 mon.a (mon.0) 1367 : cluster 0 osdmap e795: 8 total, 8 up, 8 in 2024-03-07T03:29:55.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:55 smithi189 bash[66969]: audit 2024-03-07T03:29:54.517639+0000 mon.a (mon.0) 1366 : audit 1 from='client.? 172.21.15.145:0/3123636824' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:55.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:55 smithi189 bash[66969]: cluster 2024-03-07T03:29:54.527382+0000 mon.a (mon.0) 1367 : cluster 0 osdmap e795: 8 total, 8 up, 8 in 2024-03-07T03:29:56.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:56 smithi145 bash[79878]: cluster 2024-03-07T03:29:55.132143+0000 mgr.y (mgr.34107) 542 : cluster 0 pgmap v962: 713 pgs: 10 creating+peering, 18 unknown, 685 active+clean; 2.6 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 2.2 KiB/s wr, 55 op/s 2024-03-07T03:29:56.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:56 smithi145 bash[79878]: cluster 2024-03-07T03:29:55.557145+0000 mon.a (mon.0) 1368 : cluster 0 osdmap e796: 8 total, 8 up, 8 in 2024-03-07T03:29:56.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:56 smithi145 bash[82329]: cluster 2024-03-07T03:29:55.132143+0000 mgr.y (mgr.34107) 542 : cluster 0 pgmap v962: 713 pgs: 10 creating+peering, 18 unknown, 685 active+clean; 2.6 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 2.2 KiB/s wr, 55 op/s 2024-03-07T03:29:56.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:56 smithi145 bash[82329]: cluster 2024-03-07T03:29:55.557145+0000 mon.a (mon.0) 1368 : cluster 0 osdmap e796: 8 total, 8 up, 8 in 2024-03-07T03:29:56.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:56 smithi189 bash[66969]: cluster 2024-03-07T03:29:55.132143+0000 mgr.y (mgr.34107) 542 : cluster 0 pgmap v962: 713 pgs: 10 creating+peering, 18 unknown, 685 active+clean; 2.6 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 2.2 KiB/s wr, 55 op/s 2024-03-07T03:29:56.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:56 smithi189 bash[66969]: cluster 2024-03-07T03:29:55.557145+0000 mon.a (mon.0) 1368 : cluster 0 osdmap e796: 8 total, 8 up, 8 in 2024-03-07T03:29:57.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:57 smithi189 bash[66969]: cluster 2024-03-07T03:29:56.568041+0000 mon.a (mon.0) 1369 : cluster 0 osdmap e797: 8 total, 8 up, 8 in 2024-03-07T03:29:57.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:57 smithi189 bash[66969]: cluster 2024-03-07T03:29:57.518876+0000 mon.a (mon.0) 1370 : cluster 0 osdmap e798: 8 total, 8 up, 8 in 2024-03-07T03:29:58.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:57 smithi145 bash[79878]: cluster 2024-03-07T03:29:56.568041+0000 mon.a (mon.0) 1369 : cluster 0 osdmap e797: 8 total, 8 up, 8 in 2024-03-07T03:29:58.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:57 smithi145 bash[79878]: cluster 2024-03-07T03:29:57.518876+0000 mon.a (mon.0) 1370 : cluster 0 osdmap e798: 8 total, 8 up, 8 in 2024-03-07T03:29:58.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:57 smithi145 bash[82329]: cluster 2024-03-07T03:29:56.568041+0000 mon.a (mon.0) 1369 : cluster 0 osdmap e797: 8 total, 8 up, 8 in 2024-03-07T03:29:58.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:57 smithi145 bash[82329]: cluster 2024-03-07T03:29:57.518876+0000 mon.a (mon.0) 1370 : cluster 0 osdmap e798: 8 total, 8 up, 8 in 2024-03-07T03:29:59.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:58 smithi145 bash[79878]: cluster 2024-03-07T03:29:57.134789+0000 mgr.y (mgr.34107) 543 : cluster 0 pgmap v965: 713 pgs: 10 creating+peering, 703 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 5.0 KiB/s wr, 91 op/s 2024-03-07T03:29:59.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:58 smithi145 bash[82329]: cluster 2024-03-07T03:29:57.134789+0000 mgr.y (mgr.34107) 543 : cluster 0 pgmap v965: 713 pgs: 10 creating+peering, 703 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 5.0 KiB/s wr, 91 op/s 2024-03-07T03:29:59.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:58 smithi189 bash[66969]: cluster 2024-03-07T03:29:57.134789+0000 mgr.y (mgr.34107) 543 : cluster 0 pgmap v965: 713 pgs: 10 creating+peering, 703 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s rd, 5.0 KiB/s wr, 91 op/s 2024-03-07T03:29:59.987 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:29:59 smithi145 bash[79878]: cluster 2024-03-07T03:29:58.588590+0000 mon.a (mon.0) 1371 : cluster 0 osdmap e799: 8 total, 8 up, 8 in 2024-03-07T03:29:59.988 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:29:59 smithi145 bash[82329]: cluster 2024-03-07T03:29:58.588590+0000 mon.a (mon.0) 1371 : cluster 0 osdmap e799: 8 total, 8 up, 8 in 2024-03-07T03:30:00.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:29:59 smithi189 bash[66969]: cluster 2024-03-07T03:29:58.588590+0000 mon.a (mon.0) 1371 : cluster 0 osdmap e799: 8 total, 8 up, 8 in 2024-03-07T03:30:00.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:00 smithi145 bash[79878]: debug 2024-03-07T03:29:59.998+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:30:01.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:00 smithi145 bash[79878]: cluster 2024-03-07T03:29:59.137729+0000 mgr.y (mgr.34107) 544 : cluster 0 pgmap v968: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 5.0 KiB/s wr, 77 op/s 2024-03-07T03:30:01.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:00 smithi145 bash[79878]: cluster 2024-03-07T03:29:59.611371+0000 mon.a (mon.0) 1372 : cluster 0 osdmap e800: 8 total, 8 up, 8 in 2024-03-07T03:30:01.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:00 smithi145 bash[79878]: cluster 2024-03-07T03:30:00.000167+0000 mon.a (mon.0) 1373 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:30:01.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:00 smithi145 bash[82329]: cluster 2024-03-07T03:29:59.137729+0000 mgr.y (mgr.34107) 544 : cluster 0 pgmap v968: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 5.0 KiB/s wr, 77 op/s 2024-03-07T03:30:01.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:00 smithi145 bash[82329]: cluster 2024-03-07T03:29:59.611371+0000 mon.a (mon.0) 1372 : cluster 0 osdmap e800: 8 total, 8 up, 8 in 2024-03-07T03:30:01.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:00 smithi145 bash[82329]: cluster 2024-03-07T03:30:00.000167+0000 mon.a (mon.0) 1373 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:30:01.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:00 smithi189 bash[66969]: cluster 2024-03-07T03:29:59.137729+0000 mgr.y (mgr.34107) 544 : cluster 0 pgmap v968: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 5.0 KiB/s wr, 77 op/s 2024-03-07T03:30:01.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:00 smithi189 bash[66969]: cluster 2024-03-07T03:29:59.611371+0000 mon.a (mon.0) 1372 : cluster 0 osdmap e800: 8 total, 8 up, 8 in 2024-03-07T03:30:01.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:00 smithi189 bash[66969]: cluster 2024-03-07T03:30:00.000167+0000 mon.a (mon.0) 1373 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:30:02.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:01 smithi145 bash[79878]: cluster 2024-03-07T03:30:00.621242+0000 mon.a (mon.0) 1374 : cluster 0 osdmap e801: 8 total, 8 up, 8 in 2024-03-07T03:30:02.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:01 smithi145 bash[82329]: cluster 2024-03-07T03:30:00.621242+0000 mon.a (mon.0) 1374 : cluster 0 osdmap e801: 8 total, 8 up, 8 in 2024-03-07T03:30:02.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:01 smithi189 bash[66969]: cluster 2024-03-07T03:30:00.621242+0000 mon.a (mon.0) 1374 : cluster 0 osdmap e801: 8 total, 8 up, 8 in 2024-03-07T03:30:02.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:30:02 smithi189 bash[59814]: ts=2024-03-07T03:30:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:30:02.668 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.QuiesceWatch (10098 ms) 2024-03-07T03:30:02.668 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.QuiesceWatchPP 2024-03-07T03:30:02.669 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:30:03.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:30:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:30:02] "GET /metrics HTTP/1.1" 200 82217 "" "Prometheus/2.43.0" 2024-03-07T03:30:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:02 smithi145 bash[79878]: cluster 2024-03-07T03:30:01.140482+0000 mgr.y (mgr.34107) 545 : cluster 0 pgmap v971: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 2.7 KiB/s wr, 45 op/s 2024-03-07T03:30:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:02 smithi145 bash[79878]: cluster 2024-03-07T03:30:01.635550+0000 mon.a (mon.0) 1375 : cluster 0 osdmap e802: 8 total, 8 up, 8 in 2024-03-07T03:30:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:02 smithi145 bash[79878]: cluster 2024-03-07T03:30:02.525101+0000 mon.a (mon.0) 1376 : cluster 0 osdmap e803: 8 total, 8 up, 8 in 2024-03-07T03:30:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:02 smithi145 bash[82329]: cluster 2024-03-07T03:30:01.140482+0000 mgr.y (mgr.34107) 545 : cluster 0 pgmap v971: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 2.7 KiB/s wr, 45 op/s 2024-03-07T03:30:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:02 smithi145 bash[82329]: cluster 2024-03-07T03:30:01.635550+0000 mon.a (mon.0) 1375 : cluster 0 osdmap e802: 8 total, 8 up, 8 in 2024-03-07T03:30:03.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:02 smithi145 bash[82329]: cluster 2024-03-07T03:30:02.525101+0000 mon.a (mon.0) 1376 : cluster 0 osdmap e803: 8 total, 8 up, 8 in 2024-03-07T03:30:03.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:02 smithi189 bash[66969]: cluster 2024-03-07T03:30:01.140482+0000 mgr.y (mgr.34107) 545 : cluster 0 pgmap v971: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 2.7 KiB/s wr, 45 op/s 2024-03-07T03:30:03.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:02 smithi189 bash[66969]: cluster 2024-03-07T03:30:01.635550+0000 mon.a (mon.0) 1375 : cluster 0 osdmap e802: 8 total, 8 up, 8 in 2024-03-07T03:30:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:02 smithi189 bash[66969]: cluster 2024-03-07T03:30:02.525101+0000 mon.a (mon.0) 1376 : cluster 0 osdmap e803: 8 total, 8 up, 8 in 2024-03-07T03:30:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:03 smithi145 bash[79878]: audit 2024-03-07T03:30:02.656165+0000 mon.a (mon.0) 1377 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:03 smithi145 bash[79878]: audit 2024-03-07T03:30:02.657891+0000 mon.a (mon.0) 1378 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:04.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:03 smithi145 bash[82329]: audit 2024-03-07T03:30:02.656165+0000 mon.a (mon.0) 1377 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:04.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:03 smithi145 bash[82329]: audit 2024-03-07T03:30:02.657891+0000 mon.a (mon.0) 1378 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:03 smithi189 bash[66969]: audit 2024-03-07T03:30:02.656165+0000 mon.a (mon.0) 1377 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:03 smithi189 bash[66969]: audit 2024-03-07T03:30:02.657891+0000 mon.a (mon.0) 1378 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:05.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:04 smithi145 bash[79878]: cluster 2024-03-07T03:30:03.141677+0000 mgr.y (mgr.34107) 546 : cluster 0 pgmap v974: 713 pgs: 713 active+clean; 2.6 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 1022 B/s wr, 7 op/s 2024-03-07T03:30:05.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:04 smithi145 bash[79878]: cluster 2024-03-07T03:30:03.675114+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e804: 8 total, 8 up, 8 in 2024-03-07T03:30:05.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:04 smithi145 bash[79878]: audit 2024-03-07T03:30:04.361896+0000 mon.a (mon.0) 1380 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:04 smithi145 bash[82329]: cluster 2024-03-07T03:30:03.141677+0000 mgr.y (mgr.34107) 546 : cluster 0 pgmap v974: 713 pgs: 713 active+clean; 2.6 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 1022 B/s wr, 7 op/s 2024-03-07T03:30:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:04 smithi145 bash[82329]: cluster 2024-03-07T03:30:03.675114+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e804: 8 total, 8 up, 8 in 2024-03-07T03:30:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:04 smithi145 bash[82329]: audit 2024-03-07T03:30:04.361896+0000 mon.a (mon.0) 1380 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:05.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:04 smithi189 bash[66969]: cluster 2024-03-07T03:30:03.141677+0000 mgr.y (mgr.34107) 546 : cluster 0 pgmap v974: 713 pgs: 713 active+clean; 2.6 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 1022 B/s wr, 7 op/s 2024-03-07T03:30:05.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:04 smithi189 bash[66969]: cluster 2024-03-07T03:30:03.675114+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e804: 8 total, 8 up, 8 in 2024-03-07T03:30:05.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:04 smithi189 bash[66969]: audit 2024-03-07T03:30:04.361896+0000 mon.a (mon.0) 1380 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:05 smithi145 bash[79878]: cluster 2024-03-07T03:30:04.677689+0000 mon.a (mon.0) 1381 : cluster 0 osdmap e805: 8 total, 8 up, 8 in 2024-03-07T03:30:06.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:05 smithi145 bash[82329]: cluster 2024-03-07T03:30:04.677689+0000 mon.a (mon.0) 1381 : cluster 0 osdmap e805: 8 total, 8 up, 8 in 2024-03-07T03:30:06.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:05 smithi189 bash[66969]: cluster 2024-03-07T03:30:04.677689+0000 mon.a (mon.0) 1381 : cluster 0 osdmap e805: 8 total, 8 up, 8 in 2024-03-07T03:30:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[79878]: cluster 2024-03-07T03:30:05.143590+0000 mgr.y (mgr.34107) 547 : cluster 0 pgmap v977: 713 pgs: 713 active+clean; 2.6 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 5.0 KiB/s wr, 73 op/s 2024-03-07T03:30:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[79878]: cluster 2024-03-07T03:30:05.702193+0000 mon.a (mon.0) 1382 : cluster 0 osdmap e806: 8 total, 8 up, 8 in 2024-03-07T03:30:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[79878]: audit 2024-03-07T03:30:05.703619+0000 mon.a (mon.0) 1383 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.1f", "id": [6, 7]}]: dispatch 2024-03-07T03:30:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[79878]: audit 2024-03-07T03:30:05.704175+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]: dispatch 2024-03-07T03:30:07.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[79878]: audit 2024-03-07T03:30:05.704736+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.16", "id": [0, 3]}]: dispatch 2024-03-07T03:30:07.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[79878]: audit 2024-03-07T03:30:05.705167+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]: dispatch 2024-03-07T03:30:07.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[79878]: audit 2024-03-07T03:30:05.705563+0000 mon.a (mon.0) 1387 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.16", "id": [0, 1]}]: dispatch 2024-03-07T03:30:07.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[79878]: audit 2024-03-07T03:30:05.705949+0000 mon.a (mon.0) 1388 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1", "id": [3, 2]}]: dispatch 2024-03-07T03:30:07.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[79878]: audit 2024-03-07T03:30:05.706563+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.4", "id": [0, 1]}]: dispatch 2024-03-07T03:30:07.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[79878]: audit 2024-03-07T03:30:05.706959+0000 mon.a (mon.0) 1390 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]: dispatch 2024-03-07T03:30:07.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[79878]: audit 2024-03-07T03:30:05.707385+0000 mon.a (mon.0) 1391 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]: dispatch 2024-03-07T03:30:07.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[82329]: cluster 2024-03-07T03:30:05.143590+0000 mgr.y (mgr.34107) 547 : cluster 0 pgmap v977: 713 pgs: 713 active+clean; 2.6 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 5.0 KiB/s wr, 73 op/s 2024-03-07T03:30:07.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[82329]: cluster 2024-03-07T03:30:05.702193+0000 mon.a (mon.0) 1382 : cluster 0 osdmap e806: 8 total, 8 up, 8 in 2024-03-07T03:30:07.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[82329]: audit 2024-03-07T03:30:05.703619+0000 mon.a (mon.0) 1383 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.1f", "id": [6, 7]}]: dispatch 2024-03-07T03:30:07.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[82329]: audit 2024-03-07T03:30:05.704175+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]: dispatch 2024-03-07T03:30:07.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[82329]: audit 2024-03-07T03:30:05.704736+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.16", "id": [0, 3]}]: dispatch 2024-03-07T03:30:07.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[82329]: audit 2024-03-07T03:30:05.705167+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]: dispatch 2024-03-07T03:30:07.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[82329]: audit 2024-03-07T03:30:05.705563+0000 mon.a (mon.0) 1387 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.16", "id": [0, 1]}]: dispatch 2024-03-07T03:30:07.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[82329]: audit 2024-03-07T03:30:05.705949+0000 mon.a (mon.0) 1388 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1", "id": [3, 2]}]: dispatch 2024-03-07T03:30:07.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[82329]: audit 2024-03-07T03:30:05.706563+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.4", "id": [0, 1]}]: dispatch 2024-03-07T03:30:07.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[82329]: audit 2024-03-07T03:30:05.706959+0000 mon.a (mon.0) 1390 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]: dispatch 2024-03-07T03:30:07.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:06 smithi145 bash[82329]: audit 2024-03-07T03:30:05.707385+0000 mon.a (mon.0) 1391 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]: dispatch 2024-03-07T03:30:07.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:06 smithi189 bash[66969]: cluster 2024-03-07T03:30:05.143590+0000 mgr.y (mgr.34107) 547 : cluster 0 pgmap v977: 713 pgs: 713 active+clean; 2.6 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 5.0 KiB/s wr, 73 op/s 2024-03-07T03:30:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:06 smithi189 bash[66969]: cluster 2024-03-07T03:30:05.702193+0000 mon.a (mon.0) 1382 : cluster 0 osdmap e806: 8 total, 8 up, 8 in 2024-03-07T03:30:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:06 smithi189 bash[66969]: audit 2024-03-07T03:30:05.703619+0000 mon.a (mon.0) 1383 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.1f", "id": [6, 7]}]: dispatch 2024-03-07T03:30:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:06 smithi189 bash[66969]: audit 2024-03-07T03:30:05.704175+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]: dispatch 2024-03-07T03:30:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:06 smithi189 bash[66969]: audit 2024-03-07T03:30:05.704736+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.16", "id": [0, 3]}]: dispatch 2024-03-07T03:30:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:06 smithi189 bash[66969]: audit 2024-03-07T03:30:05.705167+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]: dispatch 2024-03-07T03:30:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:06 smithi189 bash[66969]: audit 2024-03-07T03:30:05.705563+0000 mon.a (mon.0) 1387 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.16", "id": [0, 1]}]: dispatch 2024-03-07T03:30:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:06 smithi189 bash[66969]: audit 2024-03-07T03:30:05.705949+0000 mon.a (mon.0) 1388 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1", "id": [3, 2]}]: dispatch 2024-03-07T03:30:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:06 smithi189 bash[66969]: audit 2024-03-07T03:30:05.706563+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.4", "id": [0, 1]}]: dispatch 2024-03-07T03:30:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:06 smithi189 bash[66969]: audit 2024-03-07T03:30:05.706959+0000 mon.a (mon.0) 1390 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]: dispatch 2024-03-07T03:30:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:06 smithi189 bash[66969]: audit 2024-03-07T03:30:05.707385+0000 mon.a (mon.0) 1391 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]: dispatch 2024-03-07T03:30:08.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[79878]: audit 2024-03-07T03:30:06.697678+0000 mon.a (mon.0) 1392 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.1f", "id": [6, 7]}]': finished 2024-03-07T03:30:08.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[79878]: audit 2024-03-07T03:30:06.697813+0000 mon.a (mon.0) 1393 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]': finished 2024-03-07T03:30:08.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[79878]: audit 2024-03-07T03:30:06.698016+0000 mon.a (mon.0) 1394 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.16", "id": [0, 3]}]': finished 2024-03-07T03:30:08.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[79878]: audit 2024-03-07T03:30:06.698286+0000 mon.a (mon.0) 1395 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]': finished 2024-03-07T03:30:08.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[79878]: audit 2024-03-07T03:30:06.698492+0000 mon.a (mon.0) 1396 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.16", "id": [0, 1]}]': finished 2024-03-07T03:30:08.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[79878]: audit 2024-03-07T03:30:06.698684+0000 mon.a (mon.0) 1397 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1", "id": [3, 2]}]': finished 2024-03-07T03:30:08.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[79878]: audit 2024-03-07T03:30:06.698880+0000 mon.a (mon.0) 1398 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.4", "id": [0, 1]}]': finished 2024-03-07T03:30:08.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[79878]: audit 2024-03-07T03:30:06.699082+0000 mon.a (mon.0) 1399 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-03-07T03:30:08.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[79878]: audit 2024-03-07T03:30:06.699256+0000 mon.a (mon.0) 1400 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]': finished 2024-03-07T03:30:08.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[79878]: cluster 2024-03-07T03:30:06.706037+0000 mon.a (mon.0) 1401 : cluster 0 osdmap e807: 8 total, 8 up, 8 in 2024-03-07T03:30:08.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[82329]: audit 2024-03-07T03:30:06.697678+0000 mon.a (mon.0) 1392 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.1f", "id": [6, 7]}]': finished 2024-03-07T03:30:08.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[82329]: audit 2024-03-07T03:30:06.697813+0000 mon.a (mon.0) 1393 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]': finished 2024-03-07T03:30:08.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[82329]: audit 2024-03-07T03:30:06.698016+0000 mon.a (mon.0) 1394 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.16", "id": [0, 3]}]': finished 2024-03-07T03:30:08.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[82329]: audit 2024-03-07T03:30:06.698286+0000 mon.a (mon.0) 1395 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]': finished 2024-03-07T03:30:08.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[82329]: audit 2024-03-07T03:30:06.698492+0000 mon.a (mon.0) 1396 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.16", "id": [0, 1]}]': finished 2024-03-07T03:30:08.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[82329]: audit 2024-03-07T03:30:06.698684+0000 mon.a (mon.0) 1397 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1", "id": [3, 2]}]': finished 2024-03-07T03:30:08.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[82329]: audit 2024-03-07T03:30:06.698880+0000 mon.a (mon.0) 1398 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.4", "id": [0, 1]}]': finished 2024-03-07T03:30:08.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[82329]: audit 2024-03-07T03:30:06.699082+0000 mon.a (mon.0) 1399 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-03-07T03:30:08.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[82329]: audit 2024-03-07T03:30:06.699256+0000 mon.a (mon.0) 1400 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]': finished 2024-03-07T03:30:08.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:07 smithi145 bash[82329]: cluster 2024-03-07T03:30:06.706037+0000 mon.a (mon.0) 1401 : cluster 0 osdmap e807: 8 total, 8 up, 8 in 2024-03-07T03:30:08.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:07 smithi189 bash[66969]: audit 2024-03-07T03:30:06.697678+0000 mon.a (mon.0) 1392 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.1f", "id": [6, 7]}]': finished 2024-03-07T03:30:08.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:07 smithi189 bash[66969]: audit 2024-03-07T03:30:06.697813+0000 mon.a (mon.0) 1393 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.2", "id": [0, 3]}]': finished 2024-03-07T03:30:08.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:07 smithi189 bash[66969]: audit 2024-03-07T03:30:06.698016+0000 mon.a (mon.0) 1394 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.16", "id": [0, 3]}]': finished 2024-03-07T03:30:08.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:07 smithi189 bash[66969]: audit 2024-03-07T03:30:06.698286+0000 mon.a (mon.0) 1395 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]': finished 2024-03-07T03:30:08.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:07 smithi189 bash[66969]: audit 2024-03-07T03:30:06.698492+0000 mon.a (mon.0) 1396 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.16", "id": [0, 1]}]': finished 2024-03-07T03:30:08.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:07 smithi189 bash[66969]: audit 2024-03-07T03:30:06.698684+0000 mon.a (mon.0) 1397 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1", "id": [3, 2]}]': finished 2024-03-07T03:30:08.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:07 smithi189 bash[66969]: audit 2024-03-07T03:30:06.698880+0000 mon.a (mon.0) 1398 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.4", "id": [0, 1]}]': finished 2024-03-07T03:30:08.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:07 smithi189 bash[66969]: audit 2024-03-07T03:30:06.699082+0000 mon.a (mon.0) 1399 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-03-07T03:30:08.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:07 smithi189 bash[66969]: audit 2024-03-07T03:30:06.699256+0000 mon.a (mon.0) 1400 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]': finished 2024-03-07T03:30:08.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:07 smithi189 bash[66969]: cluster 2024-03-07T03:30:06.706037+0000 mon.a (mon.0) 1401 : cluster 0 osdmap e807: 8 total, 8 up, 8 in 2024-03-07T03:30:09.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:08 smithi145 bash[79878]: cluster 2024-03-07T03:30:07.145486+0000 mgr.y (mgr.34107) 548 : cluster 0 pgmap v980: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 8.2 KiB/s wr, 133 op/s 2024-03-07T03:30:09.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:08 smithi145 bash[79878]: cluster 2024-03-07T03:30:07.736536+0000 mon.a (mon.0) 1402 : cluster 0 osdmap e808: 8 total, 8 up, 8 in 2024-03-07T03:30:09.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:08 smithi145 bash[82329]: cluster 2024-03-07T03:30:07.145486+0000 mgr.y (mgr.34107) 548 : cluster 0 pgmap v980: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 8.2 KiB/s wr, 133 op/s 2024-03-07T03:30:09.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:08 smithi145 bash[82329]: cluster 2024-03-07T03:30:07.736536+0000 mon.a (mon.0) 1402 : cluster 0 osdmap e808: 8 total, 8 up, 8 in 2024-03-07T03:30:09.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:08 smithi189 bash[66969]: cluster 2024-03-07T03:30:07.145486+0000 mgr.y (mgr.34107) 548 : cluster 0 pgmap v980: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s rd, 8.2 KiB/s wr, 133 op/s 2024-03-07T03:30:09.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:08 smithi189 bash[66969]: cluster 2024-03-07T03:30:07.736536+0000 mon.a (mon.0) 1402 : cluster 0 osdmap e808: 8 total, 8 up, 8 in 2024-03-07T03:30:09.963 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.QuiesceWatchPP (7295 ms) 2024-03-07T03:30:09.963 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.QuiesceWatchError 2024-03-07T03:30:09.963 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:30:10.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:09 smithi145 bash[79878]: cluster 2024-03-07T03:30:08.732695+0000 mon.a (mon.0) 1403 : cluster 0 osdmap e809: 8 total, 8 up, 8 in 2024-03-07T03:30:10.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:09 smithi145 bash[82329]: cluster 2024-03-07T03:30:08.732695+0000 mon.a (mon.0) 1403 : cluster 0 osdmap e809: 8 total, 8 up, 8 in 2024-03-07T03:30:10.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:09 smithi189 bash[66969]: cluster 2024-03-07T03:30:08.732695+0000 mon.a (mon.0) 1403 : cluster 0 osdmap e809: 8 total, 8 up, 8 in 2024-03-07T03:30:11.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:10 smithi145 bash[79878]: cluster 2024-03-07T03:30:09.148110+0000 mgr.y (mgr.34107) 549 : cluster 0 pgmap v983: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 61 op/s 2024-03-07T03:30:11.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:10 smithi145 bash[79878]: cluster 2024-03-07T03:30:09.758838+0000 mon.a (mon.0) 1404 : cluster 0 osdmap e810: 8 total, 8 up, 8 in 2024-03-07T03:30:11.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:10 smithi145 bash[82329]: cluster 2024-03-07T03:30:09.148110+0000 mgr.y (mgr.34107) 549 : cluster 0 pgmap v983: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 61 op/s 2024-03-07T03:30:11.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:10 smithi145 bash[82329]: cluster 2024-03-07T03:30:09.758838+0000 mon.a (mon.0) 1404 : cluster 0 osdmap e810: 8 total, 8 up, 8 in 2024-03-07T03:30:11.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:10 smithi189 bash[66969]: cluster 2024-03-07T03:30:09.148110+0000 mgr.y (mgr.34107) 549 : cluster 0 pgmap v983: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.5 KiB/s wr, 61 op/s 2024-03-07T03:30:11.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:10 smithi189 bash[66969]: cluster 2024-03-07T03:30:09.758838+0000 mon.a (mon.0) 1404 : cluster 0 osdmap e810: 8 total, 8 up, 8 in 2024-03-07T03:30:12.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:11 smithi145 bash[79878]: cluster 2024-03-07T03:30:10.769715+0000 mon.a (mon.0) 1405 : cluster 0 osdmap e811: 8 total, 8 up, 8 in 2024-03-07T03:30:12.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:11 smithi145 bash[82329]: cluster 2024-03-07T03:30:10.769715+0000 mon.a (mon.0) 1405 : cluster 0 osdmap e811: 8 total, 8 up, 8 in 2024-03-07T03:30:12.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:11 smithi189 bash[66969]: cluster 2024-03-07T03:30:10.769715+0000 mon.a (mon.0) 1405 : cluster 0 osdmap e811: 8 total, 8 up, 8 in 2024-03-07T03:30:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:30:12 smithi189 bash[59814]: ts=2024-03-07T03:30:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:30:13.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:30:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:30:12] "GET /metrics HTTP/1.1" 200 82217 "" "Prometheus/2.43.0" 2024-03-07T03:30:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:12 smithi145 bash[82329]: cluster 2024-03-07T03:30:11.150849+0000 mgr.y (mgr.34107) 550 : cluster 0 pgmap v986: 713 pgs: 713 active+clean; 2.6 MiB data, 483 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 1.5 KiB/s wr, 34 op/s; 8 B/s, 0 objects/s recovering 2024-03-07T03:30:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:12 smithi145 bash[82329]: cluster 2024-03-07T03:30:11.785020+0000 mon.a (mon.0) 1406 : cluster 0 osdmap e812: 8 total, 8 up, 8 in 2024-03-07T03:30:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:12 smithi145 bash[82329]: cluster 2024-03-07T03:30:12.527287+0000 mon.a (mon.0) 1407 : cluster 0 osdmap e813: 8 total, 8 up, 8 in 2024-03-07T03:30:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:12 smithi145 bash[79878]: cluster 2024-03-07T03:30:11.150849+0000 mgr.y (mgr.34107) 550 : cluster 0 pgmap v986: 713 pgs: 713 active+clean; 2.6 MiB data, 483 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 1.5 KiB/s wr, 34 op/s; 8 B/s, 0 objects/s recovering 2024-03-07T03:30:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:12 smithi145 bash[79878]: cluster 2024-03-07T03:30:11.785020+0000 mon.a (mon.0) 1406 : cluster 0 osdmap e812: 8 total, 8 up, 8 in 2024-03-07T03:30:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:12 smithi145 bash[79878]: cluster 2024-03-07T03:30:12.527287+0000 mon.a (mon.0) 1407 : cluster 0 osdmap e813: 8 total, 8 up, 8 in 2024-03-07T03:30:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:12 smithi189 bash[66969]: cluster 2024-03-07T03:30:11.150849+0000 mgr.y (mgr.34107) 550 : cluster 0 pgmap v986: 713 pgs: 713 active+clean; 2.6 MiB data, 483 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 1.5 KiB/s wr, 34 op/s; 8 B/s, 0 objects/s recovering 2024-03-07T03:30:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:12 smithi189 bash[66969]: cluster 2024-03-07T03:30:11.785020+0000 mon.a (mon.0) 1406 : cluster 0 osdmap e812: 8 total, 8 up, 8 in 2024-03-07T03:30:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:12 smithi189 bash[66969]: cluster 2024-03-07T03:30:12.527287+0000 mon.a (mon.0) 1407 : cluster 0 osdmap e813: 8 total, 8 up, 8 in 2024-03-07T03:30:14.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:14 smithi145 bash[79878]: cluster 2024-03-07T03:30:13.152054+0000 mgr.y (mgr.34107) 551 : cluster 0 pgmap v989: 713 pgs: 713 active+clean; 2.6 MiB data, 487 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 4.5 KiB/s wr, 86 op/s; 8 B/s, 2 keys/s, 0 objects/s recovering 2024-03-07T03:30:14.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:14 smithi145 bash[79878]: cluster 2024-03-07T03:30:13.531336+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e814: 8 total, 8 up, 8 in 2024-03-07T03:30:14.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:14 smithi145 bash[82329]: cluster 2024-03-07T03:30:13.152054+0000 mgr.y (mgr.34107) 551 : cluster 0 pgmap v989: 713 pgs: 713 active+clean; 2.6 MiB data, 487 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 4.5 KiB/s wr, 86 op/s; 8 B/s, 2 keys/s, 0 objects/s recovering 2024-03-07T03:30:14.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:14 smithi145 bash[82329]: cluster 2024-03-07T03:30:13.531336+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e814: 8 total, 8 up, 8 in 2024-03-07T03:30:14.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:14 smithi189 bash[66969]: cluster 2024-03-07T03:30:13.152054+0000 mgr.y (mgr.34107) 551 : cluster 0 pgmap v989: 713 pgs: 713 active+clean; 2.6 MiB data, 487 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 4.5 KiB/s wr, 86 op/s; 8 B/s, 2 keys/s, 0 objects/s recovering 2024-03-07T03:30:14.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:14 smithi189 bash[66969]: cluster 2024-03-07T03:30:13.531336+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e814: 8 total, 8 up, 8 in 2024-03-07T03:30:15.771 INFO:tasks.workunit.client.0.smithi145.stdout:10.00% done 2024-03-07T03:30:15.771 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.QuiesceWatchError (5808 ms) 2024-03-07T03:30:15.771 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.QuiesceWatchTimeout 2024-03-07T03:30:15.771 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:30:15.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:15 smithi145 bash[79878]: cluster 2024-03-07T03:30:14.533836+0000 mon.a (mon.0) 1409 : cluster 0 osdmap e815: 8 total, 8 up, 8 in 2024-03-07T03:30:15.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:15 smithi145 bash[82329]: cluster 2024-03-07T03:30:14.533836+0000 mon.a (mon.0) 1409 : cluster 0 osdmap e815: 8 total, 8 up, 8 in 2024-03-07T03:30:15.810 INFO:tasks.workunit.client.0.smithi145.stderr:test quiesce is not long enough to time out 2024-03-07T03:30:15.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:15 smithi189 bash[66969]: cluster 2024-03-07T03:30:14.533836+0000 mon.a (mon.0) 1409 : cluster 0 osdmap e815: 8 total, 8 up, 8 in 2024-03-07T03:30:16.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:16 smithi145 bash[79878]: cluster 2024-03-07T03:30:15.154004+0000 mgr.y (mgr.34107) 552 : cluster 0 pgmap v992: 713 pgs: 713 active+clean; 2.6 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 3.2 KiB/s wr, 57 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-03-07T03:30:16.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:16 smithi145 bash[79878]: cluster 2024-03-07T03:30:15.560234+0000 mon.a (mon.0) 1410 : cluster 0 osdmap e816: 8 total, 8 up, 8 in 2024-03-07T03:30:16.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:16 smithi145 bash[82329]: cluster 2024-03-07T03:30:15.154004+0000 mgr.y (mgr.34107) 552 : cluster 0 pgmap v992: 713 pgs: 713 active+clean; 2.6 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 3.2 KiB/s wr, 57 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-03-07T03:30:16.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:16 smithi145 bash[82329]: cluster 2024-03-07T03:30:15.560234+0000 mon.a (mon.0) 1410 : cluster 0 osdmap e816: 8 total, 8 up, 8 in 2024-03-07T03:30:16.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:16 smithi189 bash[66969]: cluster 2024-03-07T03:30:15.154004+0000 mgr.y (mgr.34107) 552 : cluster 0 pgmap v992: 713 pgs: 713 active+clean; 2.6 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 3.2 KiB/s wr, 57 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-03-07T03:30:16.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:16 smithi189 bash[66969]: cluster 2024-03-07T03:30:15.560234+0000 mon.a (mon.0) 1410 : cluster 0 osdmap e816: 8 total, 8 up, 8 in 2024-03-07T03:30:18.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:18 smithi145 bash[79878]: cluster 2024-03-07T03:30:17.157789+0000 mgr.y (mgr.34107) 553 : cluster 0 pgmap v994: 713 pgs: 713 active+clean; 2.6 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 4.3 KiB/s wr, 81 op/s 2024-03-07T03:30:18.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:18 smithi145 bash[79878]: cluster 2024-03-07T03:30:17.530994+0000 mon.a (mon.0) 1411 : cluster 0 osdmap e817: 8 total, 8 up, 8 in 2024-03-07T03:30:18.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:18 smithi145 bash[79878]: audit 2024-03-07T03:30:17.654899+0000 mon.a (mon.0) 1412 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:18.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:18 smithi145 bash[82329]: cluster 2024-03-07T03:30:17.157789+0000 mgr.y (mgr.34107) 553 : cluster 0 pgmap v994: 713 pgs: 713 active+clean; 2.6 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 4.3 KiB/s wr, 81 op/s 2024-03-07T03:30:18.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:18 smithi145 bash[82329]: cluster 2024-03-07T03:30:17.530994+0000 mon.a (mon.0) 1411 : cluster 0 osdmap e817: 8 total, 8 up, 8 in 2024-03-07T03:30:18.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:18 smithi145 bash[82329]: audit 2024-03-07T03:30:17.654899+0000 mon.a (mon.0) 1412 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:18.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:18 smithi189 bash[66969]: cluster 2024-03-07T03:30:17.157789+0000 mgr.y (mgr.34107) 553 : cluster 0 pgmap v994: 713 pgs: 713 active+clean; 2.6 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 4.3 KiB/s wr, 81 op/s 2024-03-07T03:30:18.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:18 smithi189 bash[66969]: cluster 2024-03-07T03:30:17.530994+0000 mon.a (mon.0) 1411 : cluster 0 osdmap e817: 8 total, 8 up, 8 in 2024-03-07T03:30:18.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:18 smithi189 bash[66969]: audit 2024-03-07T03:30:17.654899+0000 mon.a (mon.0) 1412 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:20.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:20 smithi145 bash[79878]: cluster 2024-03-07T03:30:19.160735+0000 mgr.y (mgr.34107) 554 : cluster 0 pgmap v996: 713 pgs: 713 active+clean; 2.6 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 4.1 KiB/s wr, 76 op/s 2024-03-07T03:30:20.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:20 smithi145 bash[82329]: cluster 2024-03-07T03:30:19.160735+0000 mgr.y (mgr.34107) 554 : cluster 0 pgmap v996: 713 pgs: 713 active+clean; 2.6 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 4.1 KiB/s wr, 76 op/s 2024-03-07T03:30:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:20 smithi189 bash[66969]: cluster 2024-03-07T03:30:19.160735+0000 mgr.y (mgr.34107) 554 : cluster 0 pgmap v996: 713 pgs: 713 active+clean; 2.6 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 4.1 KiB/s wr, 76 op/s 2024-03-07T03:30:22.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:30:22 smithi189 bash[59814]: ts=2024-03-07T03:30:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:30:22.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:22 smithi189 bash[66969]: cluster 2024-03-07T03:30:21.164085+0000 mgr.y (mgr.34107) 555 : cluster 0 pgmap v997: 713 pgs: 713 active+clean; 2.6 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.9 KiB/s wr, 64 op/s 2024-03-07T03:30:22.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:22 smithi189 bash[66969]: cluster 2024-03-07T03:30:22.531978+0000 mon.a (mon.0) 1413 : cluster 0 osdmap e818: 8 total, 8 up, 8 in 2024-03-07T03:30:22.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:30:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:30:22] "GET /metrics HTTP/1.1" 200 82212 "" "Prometheus/2.43.0" 2024-03-07T03:30:22.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:22 smithi145 bash[79878]: cluster 2024-03-07T03:30:21.164085+0000 mgr.y (mgr.34107) 555 : cluster 0 pgmap v997: 713 pgs: 713 active+clean; 2.6 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.9 KiB/s wr, 64 op/s 2024-03-07T03:30:22.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:22 smithi145 bash[79878]: cluster 2024-03-07T03:30:22.531978+0000 mon.a (mon.0) 1413 : cluster 0 osdmap e818: 8 total, 8 up, 8 in 2024-03-07T03:30:22.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:22 smithi145 bash[82329]: cluster 2024-03-07T03:30:21.164085+0000 mgr.y (mgr.34107) 555 : cluster 0 pgmap v997: 713 pgs: 713 active+clean; 2.6 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.9 KiB/s wr, 64 op/s 2024-03-07T03:30:22.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:22 smithi145 bash[82329]: cluster 2024-03-07T03:30:22.531978+0000 mon.a (mon.0) 1413 : cluster 0 osdmap e818: 8 total, 8 up, 8 in 2024-03-07T03:30:24.604 INFO:tasks.workunit.client.0.smithi145.stderr:test quiesce is timed out 2024-03-07T03:30:24.634 INFO:tasks.workunit.client.0.smithi145.stderr:waiting for timed out ... 0 2024-03-07T03:30:24.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:24 smithi145 bash[79878]: cluster 2024-03-07T03:30:23.165423+0000 mgr.y (mgr.34107) 556 : cluster 0 pgmap v999: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 3.3 KiB/s wr, 55 op/s 2024-03-07T03:30:24.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:24 smithi145 bash[82329]: cluster 2024-03-07T03:30:23.165423+0000 mgr.y (mgr.34107) 556 : cluster 0 pgmap v999: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 3.3 KiB/s wr, 55 op/s 2024-03-07T03:30:24.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:24 smithi189 bash[66969]: cluster 2024-03-07T03:30:23.165423+0000 mgr.y (mgr.34107) 556 : cluster 0 pgmap v999: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 3.3 KiB/s wr, 55 op/s 2024-03-07T03:30:25.635 INFO:tasks.workunit.client.0.smithi145.stderr:waiting for timed out ... 1 2024-03-07T03:30:25.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:25 smithi145 bash[79878]: cluster 2024-03-07T03:30:24.550723+0000 mon.a (mon.0) 1414 : cluster 0 osdmap e819: 8 total, 8 up, 8 in 2024-03-07T03:30:25.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:25 smithi145 bash[82329]: cluster 2024-03-07T03:30:24.550723+0000 mon.a (mon.0) 1414 : cluster 0 osdmap e819: 8 total, 8 up, 8 in 2024-03-07T03:30:25.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:25 smithi189 bash[66969]: cluster 2024-03-07T03:30:24.550723+0000 mon.a (mon.0) 1414 : cluster 0 osdmap e819: 8 total, 8 up, 8 in 2024-03-07T03:30:26.635 INFO:tasks.workunit.client.0.smithi145.stderr:waiting for timed out ... 2 2024-03-07T03:30:26.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:26 smithi145 bash[79878]: cluster 2024-03-07T03:30:25.167420+0000 mgr.y (mgr.34107) 557 : cluster 0 pgmap v1001: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 804 B/s wr, 7 op/s 2024-03-07T03:30:26.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:26 smithi145 bash[82329]: cluster 2024-03-07T03:30:25.167420+0000 mgr.y (mgr.34107) 557 : cluster 0 pgmap v1001: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 804 B/s wr, 7 op/s 2024-03-07T03:30:26.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:26 smithi189 bash[66969]: cluster 2024-03-07T03:30:25.167420+0000 mgr.y (mgr.34107) 557 : cluster 0 pgmap v1001: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 804 B/s wr, 7 op/s 2024-03-07T03:30:27.635 INFO:tasks.workunit.client.0.smithi145.stderr:waiting for timed out ... 3 2024-03-07T03:30:28.635 INFO:tasks.workunit.client.0.smithi145.stderr:waiting for timed out ... 4 2024-03-07T03:30:28.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:28 smithi145 bash[79878]: cluster 2024-03-07T03:30:27.171162+0000 mgr.y (mgr.34107) 558 : cluster 0 pgmap v1002: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 383 B/s wr, 1 op/s 2024-03-07T03:30:28.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:28 smithi145 bash[82329]: cluster 2024-03-07T03:30:27.171162+0000 mgr.y (mgr.34107) 558 : cluster 0 pgmap v1002: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 383 B/s wr, 1 op/s 2024-03-07T03:30:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:28 smithi189 bash[66969]: cluster 2024-03-07T03:30:27.171162+0000 mgr.y (mgr.34107) 558 : cluster 0 pgmap v1002: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 383 B/s wr, 1 op/s 2024-03-07T03:30:29.635 INFO:tasks.workunit.client.0.smithi145.stderr:waiting for timed out ... 5 2024-03-07T03:30:30.635 INFO:tasks.workunit.client.0.smithi145.stderr:waiting for timed out ... 6 2024-03-07T03:30:30.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:30 smithi189 bash[66969]: cluster 2024-03-07T03:30:29.174140+0000 mgr.y (mgr.34107) 559 : cluster 0 pgmap v1003: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-03-07T03:30:31.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:30 smithi145 bash[79878]: cluster 2024-03-07T03:30:29.174140+0000 mgr.y (mgr.34107) 559 : cluster 0 pgmap v1003: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-03-07T03:30:31.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:30 smithi145 bash[82329]: cluster 2024-03-07T03:30:29.174140+0000 mgr.y (mgr.34107) 559 : cluster 0 pgmap v1003: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-03-07T03:30:31.635 INFO:tasks.workunit.client.0.smithi145.stderr:waiting for timed out ... 7 2024-03-07T03:30:32.565 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:30:32 smithi189 bash[59814]: ts=2024-03-07T03:30:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:30:32.635 INFO:tasks.workunit.client.0.smithi145.stderr:waiting for timed out ... 8 2024-03-07T03:30:32.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:32 smithi189 bash[66969]: cluster 2024-03-07T03:30:31.177447+0000 mgr.y (mgr.34107) 560 : cluster 0 pgmap v1004: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 355 B/s wr, 4 op/s 2024-03-07T03:30:32.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:32 smithi189 bash[66969]: audit 2024-03-07T03:30:32.430254+0000 mon.a (mon.0) 1415 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:30:33.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:30:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:30:32] "GET /metrics HTTP/1.1" 200 82212 "" "Prometheus/2.43.0" 2024-03-07T03:30:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:32 smithi145 bash[79878]: cluster 2024-03-07T03:30:31.177447+0000 mgr.y (mgr.34107) 560 : cluster 0 pgmap v1004: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 355 B/s wr, 4 op/s 2024-03-07T03:30:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:32 smithi145 bash[79878]: audit 2024-03-07T03:30:32.430254+0000 mon.a (mon.0) 1415 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:30:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:32 smithi145 bash[82329]: cluster 2024-03-07T03:30:31.177447+0000 mgr.y (mgr.34107) 560 : cluster 0 pgmap v1004: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 355 B/s wr, 4 op/s 2024-03-07T03:30:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:32 smithi145 bash[82329]: audit 2024-03-07T03:30:32.430254+0000 mon.a (mon.0) 1415 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:30:33.636 INFO:tasks.workunit.client.0.smithi145.stderr:waiting for timed out ... 9 2024-03-07T03:30:33.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:33 smithi189 bash[66969]: audit 2024-03-07T03:30:32.654304+0000 mon.a (mon.0) 1416 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:34.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:33 smithi145 bash[79878]: audit 2024-03-07T03:30:32.654304+0000 mon.a (mon.0) 1416 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:34.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:33 smithi145 bash[82329]: audit 2024-03-07T03:30:32.654304+0000 mon.a (mon.0) 1416 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:34.636 INFO:tasks.workunit.client.0.smithi145.stderr:waiting for timed out ... 10 2024-03-07T03:30:34.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:34 smithi189 bash[66969]: cluster 2024-03-07T03:30:33.180185+0000 mgr.y (mgr.34107) 561 : cluster 0 pgmap v1005: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 306 B/s wr, 3 op/s 2024-03-07T03:30:35.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:34 smithi145 bash[79878]: cluster 2024-03-07T03:30:33.180185+0000 mgr.y (mgr.34107) 561 : cluster 0 pgmap v1005: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 306 B/s wr, 3 op/s 2024-03-07T03:30:35.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:34 smithi145 bash[82329]: cluster 2024-03-07T03:30:33.180185+0000 mgr.y (mgr.34107) 561 : cluster 0 pgmap v1005: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 306 B/s wr, 3 op/s 2024-03-07T03:30:37.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:36 smithi145 bash[79878]: cluster 2024-03-07T03:30:35.183729+0000 mgr.y (mgr.34107) 562 : cluster 0 pgmap v1006: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 288 B/s wr, 3 op/s 2024-03-07T03:30:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:36 smithi145 bash[82329]: cluster 2024-03-07T03:30:35.183729+0000 mgr.y (mgr.34107) 562 : cluster 0 pgmap v1006: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 288 B/s wr, 3 op/s 2024-03-07T03:30:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:36 smithi189 bash[66969]: cluster 2024-03-07T03:30:35.183729+0000 mgr.y (mgr.34107) 562 : cluster 0 pgmap v1006: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 288 B/s wr, 3 op/s 2024-03-07T03:30:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:38 smithi145 bash[79878]: cluster 2024-03-07T03:30:37.187856+0000 mgr.y (mgr.34107) 563 : cluster 0 pgmap v1007: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-07T03:30:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:38 smithi145 bash[79878]: audit 2024-03-07T03:30:38.287750+0000 mon.a (mon.0) 1417 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:38 smithi145 bash[79878]: audit 2024-03-07T03:30:38.300958+0000 mon.a (mon.0) 1418 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:38 smithi145 bash[79878]: audit 2024-03-07T03:30:38.491512+0000 mon.a (mon.0) 1419 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:38 smithi145 bash[79878]: audit 2024-03-07T03:30:38.505893+0000 mon.a (mon.0) 1420 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:39.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:38 smithi145 bash[82329]: cluster 2024-03-07T03:30:37.187856+0000 mgr.y (mgr.34107) 563 : cluster 0 pgmap v1007: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-07T03:30:39.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:38 smithi145 bash[82329]: audit 2024-03-07T03:30:38.287750+0000 mon.a (mon.0) 1417 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:39.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:38 smithi145 bash[82329]: audit 2024-03-07T03:30:38.300958+0000 mon.a (mon.0) 1418 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:39.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:38 smithi145 bash[82329]: audit 2024-03-07T03:30:38.491512+0000 mon.a (mon.0) 1419 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:39.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:38 smithi145 bash[82329]: audit 2024-03-07T03:30:38.505893+0000 mon.a (mon.0) 1420 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:38 smithi189 bash[66969]: cluster 2024-03-07T03:30:37.187856+0000 mgr.y (mgr.34107) 563 : cluster 0 pgmap v1007: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-07T03:30:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:38 smithi189 bash[66969]: audit 2024-03-07T03:30:38.287750+0000 mon.a (mon.0) 1417 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:38 smithi189 bash[66969]: audit 2024-03-07T03:30:38.300958+0000 mon.a (mon.0) 1418 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:38 smithi189 bash[66969]: audit 2024-03-07T03:30:38.491512+0000 mon.a (mon.0) 1419 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:38 smithi189 bash[66969]: audit 2024-03-07T03:30:38.505893+0000 mon.a (mon.0) 1420 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:40.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:39 smithi145 bash[79878]: audit 2024-03-07T03:30:39.138482+0000 mon.a (mon.0) 1421 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:30:40.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:39 smithi145 bash[79878]: audit 2024-03-07T03:30:39.140512+0000 mon.a (mon.0) 1422 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:30:40.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:39 smithi145 bash[79878]: audit 2024-03-07T03:30:39.151468+0000 mon.a (mon.0) 1423 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:40.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:39 smithi145 bash[82329]: audit 2024-03-07T03:30:39.138482+0000 mon.a (mon.0) 1421 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:30:40.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:39 smithi145 bash[82329]: audit 2024-03-07T03:30:39.140512+0000 mon.a (mon.0) 1422 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:30:40.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:39 smithi145 bash[82329]: audit 2024-03-07T03:30:39.151468+0000 mon.a (mon.0) 1423 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:39 smithi189 bash[66969]: audit 2024-03-07T03:30:39.138482+0000 mon.a (mon.0) 1421 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:30:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:39 smithi189 bash[66969]: audit 2024-03-07T03:30:39.140512+0000 mon.a (mon.0) 1422 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:30:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:39 smithi189 bash[66969]: audit 2024-03-07T03:30:39.151468+0000 mon.a (mon.0) 1423 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:30:41.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:40 smithi145 bash[79878]: cluster 2024-03-07T03:30:39.190839+0000 mgr.y (mgr.34107) 564 : cluster 0 pgmap v1008: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-07T03:30:41.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:40 smithi145 bash[82329]: cluster 2024-03-07T03:30:39.190839+0000 mgr.y (mgr.34107) 564 : cluster 0 pgmap v1008: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-07T03:30:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:40 smithi189 bash[66969]: cluster 2024-03-07T03:30:39.190839+0000 mgr.y (mgr.34107) 564 : cluster 0 pgmap v1008: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-07T03:30:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:30:42 smithi189 bash[59814]: ts=2024-03-07T03:30:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:30:43.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:42 smithi145 bash[79878]: cluster 2024-03-07T03:30:41.194165+0000 mgr.y (mgr.34107) 565 : cluster 0 pgmap v1009: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:30:43.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:30:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:30:42] "GET /metrics HTTP/1.1" 200 82210 "" "Prometheus/2.43.0" 2024-03-07T03:30:43.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:42 smithi145 bash[82329]: cluster 2024-03-07T03:30:41.194165+0000 mgr.y (mgr.34107) 565 : cluster 0 pgmap v1009: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:30:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:42 smithi189 bash[66969]: cluster 2024-03-07T03:30:41.194165+0000 mgr.y (mgr.34107) 565 : cluster 0 pgmap v1009: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:30:45.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:44 smithi145 bash[79878]: cluster 2024-03-07T03:30:43.196781+0000 mgr.y (mgr.34107) 566 : cluster 0 pgmap v1010: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:30:45.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:44 smithi145 bash[82329]: cluster 2024-03-07T03:30:43.196781+0000 mgr.y (mgr.34107) 566 : cluster 0 pgmap v1010: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:30:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:44 smithi189 bash[66969]: cluster 2024-03-07T03:30:43.196781+0000 mgr.y (mgr.34107) 566 : cluster 0 pgmap v1010: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:30:45.636 INFO:tasks.workunit.client.0.smithi145.stderr:test retry succeeds 2024-03-07T03:30:47.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:46 smithi145 bash[79878]: cluster 2024-03-07T03:30:45.200305+0000 mgr.y (mgr.34107) 567 : cluster 0 pgmap v1011: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:30:47.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:46 smithi145 bash[82329]: cluster 2024-03-07T03:30:45.200305+0000 mgr.y (mgr.34107) 567 : cluster 0 pgmap v1011: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:30:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:46 smithi189 bash[66969]: cluster 2024-03-07T03:30:45.200305+0000 mgr.y (mgr.34107) 567 : cluster 0 pgmap v1011: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:30:48.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:47 smithi145 bash[79878]: cluster 2024-03-07T03:30:46.625348+0000 mon.a (mon.0) 1424 : cluster 0 osdmap e820: 8 total, 8 up, 8 in 2024-03-07T03:30:48.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:47 smithi145 bash[82329]: cluster 2024-03-07T03:30:46.625348+0000 mon.a (mon.0) 1424 : cluster 0 osdmap e820: 8 total, 8 up, 8 in 2024-03-07T03:30:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:47 smithi189 bash[66969]: cluster 2024-03-07T03:30:46.625348+0000 mon.a (mon.0) 1424 : cluster 0 osdmap e820: 8 total, 8 up, 8 in 2024-03-07T03:30:48.911 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.QuiesceWatchTimeout (33140 ms) 2024-03-07T03:30:48.911 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.WriteZeroes 2024-03-07T03:30:48.911 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:30:49.054 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.WriteZeroes (144 ms) 2024-03-07T03:30:49.055 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.WriteZeroesThickProvision 2024-03-07T03:30:49.055 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:30:49.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:48 smithi145 bash[82329]: cluster 2024-03-07T03:30:47.203030+0000 mgr.y (mgr.34107) 568 : cluster 0 pgmap v1013: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:30:49.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:48 smithi145 bash[82329]: cluster 2024-03-07T03:30:47.647486+0000 mon.a (mon.0) 1425 : cluster 0 osdmap e821: 8 total, 8 up, 8 in 2024-03-07T03:30:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:48 smithi145 bash[82329]: audit 2024-03-07T03:30:47.655697+0000 mon.a (mon.0) 1426 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:48 smithi145 bash[79878]: cluster 2024-03-07T03:30:47.203030+0000 mgr.y (mgr.34107) 568 : cluster 0 pgmap v1013: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:30:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:48 smithi145 bash[79878]: cluster 2024-03-07T03:30:47.647486+0000 mon.a (mon.0) 1425 : cluster 0 osdmap e821: 8 total, 8 up, 8 in 2024-03-07T03:30:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:48 smithi145 bash[79878]: audit 2024-03-07T03:30:47.655697+0000 mon.a (mon.0) 1426 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:48 smithi189 bash[66969]: cluster 2024-03-07T03:30:47.203030+0000 mgr.y (mgr.34107) 568 : cluster 0 pgmap v1013: 713 pgs: 713 active+clean; 2.6 MiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:30:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:48 smithi189 bash[66969]: cluster 2024-03-07T03:30:47.647486+0000 mon.a (mon.0) 1425 : cluster 0 osdmap e821: 8 total, 8 up, 8 in 2024-03-07T03:30:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:48 smithi189 bash[66969]: audit 2024-03-07T03:30:47.655697+0000 mon.a (mon.0) 1426 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:49.212 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.WriteZeroesThickProvision (158 ms) 2024-03-07T03:30:49.213 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] TestLibRBD.ConcurentOperations 2024-03-07T03:30:49.213 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:30:50.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:49 smithi145 bash[79878]: cluster 2024-03-07T03:30:48.663208+0000 mon.a (mon.0) 1427 : cluster 0 osdmap e822: 8 total, 8 up, 8 in 2024-03-07T03:30:50.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:49 smithi145 bash[82329]: cluster 2024-03-07T03:30:48.663208+0000 mon.a (mon.0) 1427 : cluster 0 osdmap e822: 8 total, 8 up, 8 in 2024-03-07T03:30:50.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:49 smithi189 bash[66969]: cluster 2024-03-07T03:30:48.663208+0000 mon.a (mon.0) 1427 : cluster 0 osdmap e822: 8 total, 8 up, 8 in 2024-03-07T03:30:51.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:50 smithi145 bash[79878]: cluster 2024-03-07T03:30:49.205540+0000 mgr.y (mgr.34107) 569 : cluster 0 pgmap v1016: 713 pgs: 713 active+clean; 2.6 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 1.7 KiB/s wr, 28 op/s 2024-03-07T03:30:51.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:50 smithi145 bash[79878]: cluster 2024-03-07T03:30:49.676589+0000 mon.a (mon.0) 1428 : cluster 0 osdmap e823: 8 total, 8 up, 8 in 2024-03-07T03:30:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:50 smithi145 bash[82329]: cluster 2024-03-07T03:30:49.205540+0000 mgr.y (mgr.34107) 569 : cluster 0 pgmap v1016: 713 pgs: 713 active+clean; 2.6 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 1.7 KiB/s wr, 28 op/s 2024-03-07T03:30:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:50 smithi145 bash[82329]: cluster 2024-03-07T03:30:49.676589+0000 mon.a (mon.0) 1428 : cluster 0 osdmap e823: 8 total, 8 up, 8 in 2024-03-07T03:30:51.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:50 smithi189 bash[66969]: cluster 2024-03-07T03:30:49.205540+0000 mgr.y (mgr.34107) 569 : cluster 0 pgmap v1016: 713 pgs: 713 active+clean; 2.6 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 1.7 KiB/s wr, 28 op/s 2024-03-07T03:30:51.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:50 smithi189 bash[66969]: cluster 2024-03-07T03:30:49.676589+0000 mon.a (mon.0) 1428 : cluster 0 osdmap e823: 8 total, 8 up, 8 in 2024-03-07T03:30:52.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:51 smithi145 bash[79878]: cluster 2024-03-07T03:30:50.691510+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e824: 8 total, 8 up, 8 in 2024-03-07T03:30:52.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:51 smithi145 bash[82329]: cluster 2024-03-07T03:30:50.691510+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e824: 8 total, 8 up, 8 in 2024-03-07T03:30:52.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:51 smithi189 bash[66969]: cluster 2024-03-07T03:30:50.691510+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e824: 8 total, 8 up, 8 in 2024-03-07T03:30:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:30:52 smithi189 bash[59814]: ts=2024-03-07T03:30:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:30:53.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:30:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:30:52] "GET /metrics HTTP/1.1" 200 82201 "" "Prometheus/2.43.0" 2024-03-07T03:30:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:52 smithi145 bash[79878]: cluster 2024-03-07T03:30:51.208312+0000 mgr.y (mgr.34107) 570 : cluster 0 pgmap v1019: 713 pgs: 713 active+clean; 2.6 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 5.7 KiB/s wr, 72 op/s 2024-03-07T03:30:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:52 smithi145 bash[79878]: cluster 2024-03-07T03:30:51.698865+0000 mon.a (mon.0) 1430 : cluster 0 osdmap e825: 8 total, 8 up, 8 in 2024-03-07T03:30:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:52 smithi145 bash[82329]: cluster 2024-03-07T03:30:51.208312+0000 mgr.y (mgr.34107) 570 : cluster 0 pgmap v1019: 713 pgs: 713 active+clean; 2.6 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 5.7 KiB/s wr, 72 op/s 2024-03-07T03:30:53.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:52 smithi145 bash[82329]: cluster 2024-03-07T03:30:51.698865+0000 mon.a (mon.0) 1430 : cluster 0 osdmap e825: 8 total, 8 up, 8 in 2024-03-07T03:30:53.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:52 smithi189 bash[66969]: cluster 2024-03-07T03:30:51.208312+0000 mgr.y (mgr.34107) 570 : cluster 0 pgmap v1019: 713 pgs: 713 active+clean; 2.6 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 5.7 KiB/s wr, 72 op/s 2024-03-07T03:30:53.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:52 smithi189 bash[66969]: cluster 2024-03-07T03:30:51.698865+0000 mon.a (mon.0) 1430 : cluster 0 osdmap e825: 8 total, 8 up, 8 in 2024-03-07T03:30:54.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:53 smithi145 bash[79878]: cluster 2024-03-07T03:30:52.709103+0000 mon.a (mon.0) 1431 : cluster 0 osdmap e826: 8 total, 8 up, 8 in 2024-03-07T03:30:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:53 smithi145 bash[82329]: cluster 2024-03-07T03:30:52.709103+0000 mon.a (mon.0) 1431 : cluster 0 osdmap e826: 8 total, 8 up, 8 in 2024-03-07T03:30:54.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:53 smithi189 bash[66969]: cluster 2024-03-07T03:30:52.709103+0000 mon.a (mon.0) 1431 : cluster 0 osdmap e826: 8 total, 8 up, 8 in 2024-03-07T03:30:55.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:54 smithi145 bash[79878]: cluster 2024-03-07T03:30:53.209671+0000 mgr.y (mgr.34107) 571 : cluster 0 pgmap v1022: 713 pgs: 713 active+clean; 2.6 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 4.5 KiB/s wr, 37 op/s 2024-03-07T03:30:55.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:54 smithi145 bash[79878]: cluster 2024-03-07T03:30:53.725337+0000 mon.a (mon.0) 1432 : cluster 0 osdmap e827: 8 total, 8 up, 8 in 2024-03-07T03:30:55.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:54 smithi145 bash[82329]: cluster 2024-03-07T03:30:53.209671+0000 mgr.y (mgr.34107) 571 : cluster 0 pgmap v1022: 713 pgs: 713 active+clean; 2.6 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 4.5 KiB/s wr, 37 op/s 2024-03-07T03:30:55.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:54 smithi145 bash[82329]: cluster 2024-03-07T03:30:53.725337+0000 mon.a (mon.0) 1432 : cluster 0 osdmap e827: 8 total, 8 up, 8 in 2024-03-07T03:30:55.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:54 smithi189 bash[66969]: cluster 2024-03-07T03:30:53.209671+0000 mgr.y (mgr.34107) 571 : cluster 0 pgmap v1022: 713 pgs: 713 active+clean; 2.6 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 4.5 KiB/s wr, 37 op/s 2024-03-07T03:30:55.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:54 smithi189 bash[66969]: cluster 2024-03-07T03:30:53.725337+0000 mon.a (mon.0) 1432 : cluster 0 osdmap e827: 8 total, 8 up, 8 in 2024-03-07T03:30:56.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:55 smithi145 bash[79878]: cluster 2024-03-07T03:30:54.727566+0000 mon.a (mon.0) 1433 : cluster 0 osdmap e828: 8 total, 8 up, 8 in 2024-03-07T03:30:56.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:55 smithi145 bash[82329]: cluster 2024-03-07T03:30:54.727566+0000 mon.a (mon.0) 1433 : cluster 0 osdmap e828: 8 total, 8 up, 8 in 2024-03-07T03:30:56.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:55 smithi189 bash[66969]: cluster 2024-03-07T03:30:54.727566+0000 mon.a (mon.0) 1433 : cluster 0 osdmap e828: 8 total, 8 up, 8 in 2024-03-07T03:30:57.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:56 smithi145 bash[79878]: cluster 2024-03-07T03:30:55.211829+0000 mgr.y (mgr.34107) 572 : cluster 0 pgmap v1025: 713 pgs: 713 active+clean; 2.6 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 7.0 KiB/s wr, 117 op/s 2024-03-07T03:30:57.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:56 smithi145 bash[79878]: cluster 2024-03-07T03:30:55.750584+0000 mon.a (mon.0) 1434 : cluster 0 osdmap e829: 8 total, 8 up, 8 in 2024-03-07T03:30:57.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:56 smithi145 bash[82329]: cluster 2024-03-07T03:30:55.211829+0000 mgr.y (mgr.34107) 572 : cluster 0 pgmap v1025: 713 pgs: 713 active+clean; 2.6 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 7.0 KiB/s wr, 117 op/s 2024-03-07T03:30:57.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:56 smithi145 bash[82329]: cluster 2024-03-07T03:30:55.750584+0000 mon.a (mon.0) 1434 : cluster 0 osdmap e829: 8 total, 8 up, 8 in 2024-03-07T03:30:57.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:56 smithi189 bash[66969]: cluster 2024-03-07T03:30:55.211829+0000 mgr.y (mgr.34107) 572 : cluster 0 pgmap v1025: 713 pgs: 713 active+clean; 2.6 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 7.0 KiB/s wr, 117 op/s 2024-03-07T03:30:57.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:56 smithi189 bash[66969]: cluster 2024-03-07T03:30:55.750584+0000 mon.a (mon.0) 1434 : cluster 0 osdmap e829: 8 total, 8 up, 8 in 2024-03-07T03:30:58.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:57 smithi145 bash[79878]: cluster 2024-03-07T03:30:56.752713+0000 mon.a (mon.0) 1435 : cluster 0 osdmap e830: 8 total, 8 up, 8 in 2024-03-07T03:30:58.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:57 smithi145 bash[79878]: cluster 2024-03-07T03:30:57.550319+0000 mon.a (mon.0) 1436 : cluster 0 osdmap e831: 8 total, 8 up, 8 in 2024-03-07T03:30:58.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:57 smithi145 bash[82329]: cluster 2024-03-07T03:30:56.752713+0000 mon.a (mon.0) 1435 : cluster 0 osdmap e830: 8 total, 8 up, 8 in 2024-03-07T03:30:58.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:57 smithi145 bash[82329]: cluster 2024-03-07T03:30:57.550319+0000 mon.a (mon.0) 1436 : cluster 0 osdmap e831: 8 total, 8 up, 8 in 2024-03-07T03:30:58.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:57 smithi189 bash[66969]: cluster 2024-03-07T03:30:56.752713+0000 mon.a (mon.0) 1435 : cluster 0 osdmap e830: 8 total, 8 up, 8 in 2024-03-07T03:30:58.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:57 smithi189 bash[66969]: cluster 2024-03-07T03:30:57.550319+0000 mon.a (mon.0) 1436 : cluster 0 osdmap e831: 8 total, 8 up, 8 in 2024-03-07T03:30:59.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:58 smithi145 bash[79878]: cluster 2024-03-07T03:30:57.213831+0000 mgr.y (mgr.34107) 573 : cluster 0 pgmap v1028: 713 pgs: 713 active+clean; 2.6 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 6.2 KiB/s wr, 115 op/s 2024-03-07T03:30:59.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:30:58 smithi145 bash[79878]: cluster 2024-03-07T03:30:58.550445+0000 mon.a (mon.0) 1437 : cluster 0 osdmap e832: 8 total, 8 up, 8 in 2024-03-07T03:30:59.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:58 smithi145 bash[82329]: cluster 2024-03-07T03:30:57.213831+0000 mgr.y (mgr.34107) 573 : cluster 0 pgmap v1028: 713 pgs: 713 active+clean; 2.6 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 6.2 KiB/s wr, 115 op/s 2024-03-07T03:30:59.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:30:58 smithi145 bash[82329]: cluster 2024-03-07T03:30:58.550445+0000 mon.a (mon.0) 1437 : cluster 0 osdmap e832: 8 total, 8 up, 8 in 2024-03-07T03:30:59.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:58 smithi189 bash[66969]: cluster 2024-03-07T03:30:57.213831+0000 mgr.y (mgr.34107) 573 : cluster 0 pgmap v1028: 713 pgs: 713 active+clean; 2.6 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 6.2 KiB/s wr, 115 op/s 2024-03-07T03:30:59.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:30:58 smithi189 bash[66969]: cluster 2024-03-07T03:30:58.550445+0000 mon.a (mon.0) 1437 : cluster 0 osdmap e832: 8 total, 8 up, 8 in 2024-03-07T03:31:00.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:00 smithi145 bash[79878]: cluster 2024-03-07T03:30:59.216822+0000 mgr.y (mgr.34107) 574 : cluster 0 pgmap v1031: 713 pgs: 713 active+clean; 2.6 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 175 KiB/s rd, 4.0 KiB/s wr, 237 op/s 2024-03-07T03:31:00.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:00 smithi145 bash[79878]: cluster 2024-03-07T03:30:59.555674+0000 mon.a (mon.0) 1438 : cluster 0 osdmap e833: 8 total, 8 up, 8 in 2024-03-07T03:31:00.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:00 smithi145 bash[82329]: cluster 2024-03-07T03:30:59.216822+0000 mgr.y (mgr.34107) 574 : cluster 0 pgmap v1031: 713 pgs: 713 active+clean; 2.6 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 175 KiB/s rd, 4.0 KiB/s wr, 237 op/s 2024-03-07T03:31:00.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:00 smithi145 bash[82329]: cluster 2024-03-07T03:30:59.555674+0000 mon.a (mon.0) 1438 : cluster 0 osdmap e833: 8 total, 8 up, 8 in 2024-03-07T03:31:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:00 smithi189 bash[66969]: cluster 2024-03-07T03:30:59.216822+0000 mgr.y (mgr.34107) 574 : cluster 0 pgmap v1031: 713 pgs: 713 active+clean; 2.6 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 175 KiB/s rd, 4.0 KiB/s wr, 237 op/s 2024-03-07T03:31:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:00 smithi189 bash[66969]: cluster 2024-03-07T03:30:59.555674+0000 mon.a (mon.0) 1438 : cluster 0 osdmap e833: 8 total, 8 up, 8 in 2024-03-07T03:31:01.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:01 smithi189 bash[66969]: cluster 2024-03-07T03:31:00.556676+0000 mon.a (mon.0) 1439 : cluster 0 osdmap e834: 8 total, 8 up, 8 in 2024-03-07T03:31:02.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:01 smithi145 bash[79878]: cluster 2024-03-07T03:31:00.556676+0000 mon.a (mon.0) 1439 : cluster 0 osdmap e834: 8 total, 8 up, 8 in 2024-03-07T03:31:02.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:01 smithi145 bash[82329]: cluster 2024-03-07T03:31:00.556676+0000 mon.a (mon.0) 1439 : cluster 0 osdmap e834: 8 total, 8 up, 8 in 2024-03-07T03:31:02.562 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:31:02 smithi189 bash[59814]: ts=2024-03-07T03:31:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:31:02.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:02 smithi189 bash[66969]: cluster 2024-03-07T03:31:01.219525+0000 mgr.y (mgr.34107) 575 : cluster 0 pgmap v1034: 713 pgs: 713 active+clean; 2.6 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 171 KiB/s rd, 4.0 KiB/s wr, 233 op/s 2024-03-07T03:31:02.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:02 smithi189 bash[66969]: cluster 2024-03-07T03:31:01.581657+0000 mon.a (mon.0) 1440 : cluster 0 osdmap e835: 8 total, 8 up, 8 in 2024-03-07T03:31:03.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:31:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:31:02] "GET /metrics HTTP/1.1" 200 82201 "" "Prometheus/2.43.0" 2024-03-07T03:31:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:02 smithi145 bash[82329]: cluster 2024-03-07T03:31:01.219525+0000 mgr.y (mgr.34107) 575 : cluster 0 pgmap v1034: 713 pgs: 713 active+clean; 2.6 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 171 KiB/s rd, 4.0 KiB/s wr, 233 op/s 2024-03-07T03:31:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:02 smithi145 bash[82329]: cluster 2024-03-07T03:31:01.581657+0000 mon.a (mon.0) 1440 : cluster 0 osdmap e835: 8 total, 8 up, 8 in 2024-03-07T03:31:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:02 smithi145 bash[79878]: cluster 2024-03-07T03:31:01.219525+0000 mgr.y (mgr.34107) 575 : cluster 0 pgmap v1034: 713 pgs: 713 active+clean; 2.6 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 171 KiB/s rd, 4.0 KiB/s wr, 233 op/s 2024-03-07T03:31:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:02 smithi145 bash[79878]: cluster 2024-03-07T03:31:01.581657+0000 mon.a (mon.0) 1440 : cluster 0 osdmap e835: 8 total, 8 up, 8 in 2024-03-07T03:31:04.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:03 smithi145 bash[79878]: cluster 2024-03-07T03:31:02.593585+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e836: 8 total, 8 up, 8 in 2024-03-07T03:31:04.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:03 smithi145 bash[79878]: audit 2024-03-07T03:31:02.656345+0000 mon.a (mon.0) 1442 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:04.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:03 smithi145 bash[82329]: cluster 2024-03-07T03:31:02.593585+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e836: 8 total, 8 up, 8 in 2024-03-07T03:31:04.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:03 smithi145 bash[82329]: audit 2024-03-07T03:31:02.656345+0000 mon.a (mon.0) 1442 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:03 smithi189 bash[66969]: cluster 2024-03-07T03:31:02.593585+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e836: 8 total, 8 up, 8 in 2024-03-07T03:31:04.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:03 smithi189 bash[66969]: audit 2024-03-07T03:31:02.656345+0000 mon.a (mon.0) 1442 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:05.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:04 smithi145 bash[79878]: cluster 2024-03-07T03:31:03.220730+0000 mgr.y (mgr.34107) 576 : cluster 0 pgmap v1037: 713 pgs: 713 active+clean; 2.6 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 766 B/s wr, 6 op/s 2024-03-07T03:31:05.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:04 smithi145 bash[79878]: cluster 2024-03-07T03:31:03.611718+0000 mon.a (mon.0) 1443 : cluster 0 osdmap e837: 8 total, 8 up, 8 in 2024-03-07T03:31:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:04 smithi145 bash[82329]: cluster 2024-03-07T03:31:03.220730+0000 mgr.y (mgr.34107) 576 : cluster 0 pgmap v1037: 713 pgs: 713 active+clean; 2.6 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 766 B/s wr, 6 op/s 2024-03-07T03:31:05.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:04 smithi145 bash[82329]: cluster 2024-03-07T03:31:03.611718+0000 mon.a (mon.0) 1443 : cluster 0 osdmap e837: 8 total, 8 up, 8 in 2024-03-07T03:31:05.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:04 smithi189 bash[66969]: cluster 2024-03-07T03:31:03.220730+0000 mgr.y (mgr.34107) 576 : cluster 0 pgmap v1037: 713 pgs: 713 active+clean; 2.6 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 766 B/s wr, 6 op/s 2024-03-07T03:31:05.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:04 smithi189 bash[66969]: cluster 2024-03-07T03:31:03.611718+0000 mon.a (mon.0) 1443 : cluster 0 osdmap e837: 8 total, 8 up, 8 in 2024-03-07T03:31:06.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:05 smithi145 bash[79878]: cluster 2024-03-07T03:31:04.616037+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e838: 8 total, 8 up, 8 in 2024-03-07T03:31:06.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:05 smithi145 bash[82329]: cluster 2024-03-07T03:31:04.616037+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e838: 8 total, 8 up, 8 in 2024-03-07T03:31:06.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:05 smithi189 bash[66969]: cluster 2024-03-07T03:31:04.616037+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e838: 8 total, 8 up, 8 in 2024-03-07T03:31:07.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:06 smithi145 bash[79878]: cluster 2024-03-07T03:31:05.222657+0000 mgr.y (mgr.34107) 577 : cluster 0 pgmap v1040: 713 pgs: 713 active+clean; 2.6 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 5.2 KiB/s wr, 28 op/s 2024-03-07T03:31:07.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:06 smithi145 bash[79878]: cluster 2024-03-07T03:31:05.625604+0000 mon.a (mon.0) 1445 : cluster 0 osdmap e839: 8 total, 8 up, 8 in 2024-03-07T03:31:07.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:06 smithi145 bash[82329]: cluster 2024-03-07T03:31:05.222657+0000 mgr.y (mgr.34107) 577 : cluster 0 pgmap v1040: 713 pgs: 713 active+clean; 2.6 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 5.2 KiB/s wr, 28 op/s 2024-03-07T03:31:07.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:06 smithi145 bash[82329]: cluster 2024-03-07T03:31:05.625604+0000 mon.a (mon.0) 1445 : cluster 0 osdmap e839: 8 total, 8 up, 8 in 2024-03-07T03:31:07.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:06 smithi189 bash[66969]: cluster 2024-03-07T03:31:05.222657+0000 mgr.y (mgr.34107) 577 : cluster 0 pgmap v1040: 713 pgs: 713 active+clean; 2.6 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 5.2 KiB/s wr, 28 op/s 2024-03-07T03:31:07.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:06 smithi189 bash[66969]: cluster 2024-03-07T03:31:05.625604+0000 mon.a (mon.0) 1445 : cluster 0 osdmap e839: 8 total, 8 up, 8 in 2024-03-07T03:31:08.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:07 smithi145 bash[79878]: cluster 2024-03-07T03:31:06.628113+0000 mon.a (mon.0) 1446 : cluster 0 osdmap e840: 8 total, 8 up, 8 in 2024-03-07T03:31:08.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:07 smithi145 bash[79878]: cluster 2024-03-07T03:31:07.549659+0000 mon.a (mon.0) 1447 : cluster 0 osdmap e841: 8 total, 8 up, 8 in 2024-03-07T03:31:08.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:07 smithi145 bash[82329]: cluster 2024-03-07T03:31:06.628113+0000 mon.a (mon.0) 1446 : cluster 0 osdmap e840: 8 total, 8 up, 8 in 2024-03-07T03:31:08.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:07 smithi145 bash[82329]: cluster 2024-03-07T03:31:07.549659+0000 mon.a (mon.0) 1447 : cluster 0 osdmap e841: 8 total, 8 up, 8 in 2024-03-07T03:31:08.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:07 smithi189 bash[66969]: cluster 2024-03-07T03:31:06.628113+0000 mon.a (mon.0) 1446 : cluster 0 osdmap e840: 8 total, 8 up, 8 in 2024-03-07T03:31:08.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:07 smithi189 bash[66969]: cluster 2024-03-07T03:31:07.549659+0000 mon.a (mon.0) 1447 : cluster 0 osdmap e841: 8 total, 8 up, 8 in 2024-03-07T03:31:09.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:08 smithi145 bash[79878]: cluster 2024-03-07T03:31:07.225300+0000 mgr.y (mgr.34107) 578 : cluster 0 pgmap v1043: 713 pgs: 713 active+clean; 2.6 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 5.0 KiB/s wr, 27 op/s 2024-03-07T03:31:09.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:08 smithi145 bash[82329]: cluster 2024-03-07T03:31:07.225300+0000 mgr.y (mgr.34107) 578 : cluster 0 pgmap v1043: 713 pgs: 713 active+clean; 2.6 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 5.0 KiB/s wr, 27 op/s 2024-03-07T03:31:09.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:08 smithi189 bash[66969]: cluster 2024-03-07T03:31:07.225300+0000 mgr.y (mgr.34107) 578 : cluster 0 pgmap v1043: 713 pgs: 713 active+clean; 2.6 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 5.0 KiB/s wr, 27 op/s 2024-03-07T03:31:10.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:09 smithi145 bash[79878]: cluster 2024-03-07T03:31:08.665553+0000 mon.a (mon.0) 1448 : cluster 0 osdmap e842: 8 total, 8 up, 8 in 2024-03-07T03:31:10.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:09 smithi145 bash[82329]: cluster 2024-03-07T03:31:08.665553+0000 mon.a (mon.0) 1448 : cluster 0 osdmap e842: 8 total, 8 up, 8 in 2024-03-07T03:31:10.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:09 smithi189 bash[66969]: cluster 2024-03-07T03:31:08.665553+0000 mon.a (mon.0) 1448 : cluster 0 osdmap e842: 8 total, 8 up, 8 in 2024-03-07T03:31:11.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:10 smithi145 bash[79878]: cluster 2024-03-07T03:31:09.227752+0000 mgr.y (mgr.34107) 579 : cluster 0 pgmap v1046: 713 pgs: 713 active+clean; 2.6 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 4.2 KiB/s wr, 41 op/s 2024-03-07T03:31:11.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:10 smithi145 bash[79878]: cluster 2024-03-07T03:31:09.705662+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e843: 8 total, 8 up, 8 in 2024-03-07T03:31:11.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:10 smithi145 bash[82329]: cluster 2024-03-07T03:31:09.227752+0000 mgr.y (mgr.34107) 579 : cluster 0 pgmap v1046: 713 pgs: 713 active+clean; 2.6 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 4.2 KiB/s wr, 41 op/s 2024-03-07T03:31:11.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:10 smithi145 bash[82329]: cluster 2024-03-07T03:31:09.705662+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e843: 8 total, 8 up, 8 in 2024-03-07T03:31:11.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:10 smithi189 bash[66969]: cluster 2024-03-07T03:31:09.227752+0000 mgr.y (mgr.34107) 579 : cluster 0 pgmap v1046: 713 pgs: 713 active+clean; 2.6 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 4.2 KiB/s wr, 41 op/s 2024-03-07T03:31:11.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:10 smithi189 bash[66969]: cluster 2024-03-07T03:31:09.705662+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e843: 8 total, 8 up, 8 in 2024-03-07T03:31:12.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:11 smithi145 bash[79878]: cluster 2024-03-07T03:31:10.722805+0000 mon.a (mon.0) 1450 : cluster 0 osdmap e844: 8 total, 8 up, 8 in 2024-03-07T03:31:12.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:11 smithi145 bash[82329]: cluster 2024-03-07T03:31:10.722805+0000 mon.a (mon.0) 1450 : cluster 0 osdmap e844: 8 total, 8 up, 8 in 2024-03-07T03:31:12.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:11 smithi189 bash[66969]: cluster 2024-03-07T03:31:10.722805+0000 mon.a (mon.0) 1450 : cluster 0 osdmap e844: 8 total, 8 up, 8 in 2024-03-07T03:31:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:31:12 smithi189 bash[59814]: ts=2024-03-07T03:31:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:31:13.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:31:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:31:12] "GET /metrics HTTP/1.1" 200 82229 "" "Prometheus/2.43.0" 2024-03-07T03:31:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:12 smithi145 bash[79878]: cluster 2024-03-07T03:31:11.230445+0000 mgr.y (mgr.34107) 580 : cluster 0 pgmap v1049: 713 pgs: 713 active+clean; 2.6 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 4.5 KiB/s wr, 40 op/s 2024-03-07T03:31:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:12 smithi145 bash[79878]: cluster 2024-03-07T03:31:11.745612+0000 mon.a (mon.0) 1451 : cluster 0 osdmap e845: 8 total, 8 up, 8 in 2024-03-07T03:31:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:12 smithi145 bash[79878]: cluster 2024-03-07T03:31:12.553234+0000 mon.a (mon.0) 1452 : cluster 0 osdmap e846: 8 total, 8 up, 8 in 2024-03-07T03:31:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:12 smithi145 bash[82329]: cluster 2024-03-07T03:31:11.230445+0000 mgr.y (mgr.34107) 580 : cluster 0 pgmap v1049: 713 pgs: 713 active+clean; 2.6 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 4.5 KiB/s wr, 40 op/s 2024-03-07T03:31:13.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:12 smithi145 bash[82329]: cluster 2024-03-07T03:31:11.745612+0000 mon.a (mon.0) 1451 : cluster 0 osdmap e845: 8 total, 8 up, 8 in 2024-03-07T03:31:13.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:12 smithi145 bash[82329]: cluster 2024-03-07T03:31:12.553234+0000 mon.a (mon.0) 1452 : cluster 0 osdmap e846: 8 total, 8 up, 8 in 2024-03-07T03:31:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:12 smithi189 bash[66969]: cluster 2024-03-07T03:31:11.230445+0000 mgr.y (mgr.34107) 580 : cluster 0 pgmap v1049: 713 pgs: 713 active+clean; 2.6 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 4.5 KiB/s wr, 40 op/s 2024-03-07T03:31:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:12 smithi189 bash[66969]: cluster 2024-03-07T03:31:11.745612+0000 mon.a (mon.0) 1451 : cluster 0 osdmap e845: 8 total, 8 up, 8 in 2024-03-07T03:31:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:12 smithi189 bash[66969]: cluster 2024-03-07T03:31:12.553234+0000 mon.a (mon.0) 1452 : cluster 0 osdmap e846: 8 total, 8 up, 8 in 2024-03-07T03:31:14.746 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] TestLibRBD.ConcurentOperations (25533 ms) 2024-03-07T03:31:14.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:14 smithi145 bash[79878]: cluster 2024-03-07T03:31:13.231657+0000 mgr.y (mgr.34107) 581 : cluster 0 pgmap v1052: 713 pgs: 713 active+clean; 2.6 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 511 B/s wr, 9 op/s 2024-03-07T03:31:14.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:14 smithi145 bash[79878]: cluster 2024-03-07T03:31:13.556694+0000 mon.a (mon.0) 1453 : cluster 0 osdmap e847: 8 total, 8 up, 8 in 2024-03-07T03:31:14.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:14 smithi145 bash[82329]: cluster 2024-03-07T03:31:13.231657+0000 mgr.y (mgr.34107) 581 : cluster 0 pgmap v1052: 713 pgs: 713 active+clean; 2.6 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 511 B/s wr, 9 op/s 2024-03-07T03:31:14.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:14 smithi145 bash[82329]: cluster 2024-03-07T03:31:13.556694+0000 mon.a (mon.0) 1453 : cluster 0 osdmap e847: 8 total, 8 up, 8 in 2024-03-07T03:31:14.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:14 smithi189 bash[66969]: cluster 2024-03-07T03:31:13.231657+0000 mgr.y (mgr.34107) 581 : cluster 0 pgmap v1052: 713 pgs: 713 active+clean; 2.6 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 511 B/s wr, 9 op/s 2024-03-07T03:31:14.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:14 smithi189 bash[66969]: cluster 2024-03-07T03:31:13.556694+0000 mon.a (mon.0) 1453 : cluster 0 osdmap e847: 8 total, 8 up, 8 in 2024-03-07T03:31:15.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:15 smithi189 bash[66969]: cluster 2024-03-07T03:31:14.560211+0000 mon.a (mon.0) 1454 : cluster 0 osdmap e848: 8 total, 8 up, 8 in 2024-03-07T03:31:16.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:15 smithi145 bash[79878]: cluster 2024-03-07T03:31:14.560211+0000 mon.a (mon.0) 1454 : cluster 0 osdmap e848: 8 total, 8 up, 8 in 2024-03-07T03:31:16.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:15 smithi145 bash[82329]: cluster 2024-03-07T03:31:14.560211+0000 mon.a (mon.0) 1454 : cluster 0 osdmap e848: 8 total, 8 up, 8 in 2024-03-07T03:31:16.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:16 smithi189 bash[66969]: cluster 2024-03-07T03:31:15.233571+0000 mgr.y (mgr.34107) 582 : cluster 0 pgmap v1055: 713 pgs: 713 active+clean; 2.6 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 5.5 KiB/s wr, 83 op/s 2024-03-07T03:31:16.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:16 smithi189 bash[66969]: cluster 2024-03-07T03:31:15.586844+0000 mon.a (mon.0) 1455 : cluster 0 osdmap e849: 8 total, 8 up, 8 in 2024-03-07T03:31:17.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:16 smithi145 bash[79878]: cluster 2024-03-07T03:31:15.233571+0000 mgr.y (mgr.34107) 582 : cluster 0 pgmap v1055: 713 pgs: 713 active+clean; 2.6 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 5.5 KiB/s wr, 83 op/s 2024-03-07T03:31:17.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:16 smithi145 bash[79878]: cluster 2024-03-07T03:31:15.586844+0000 mon.a (mon.0) 1455 : cluster 0 osdmap e849: 8 total, 8 up, 8 in 2024-03-07T03:31:17.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:16 smithi145 bash[82329]: cluster 2024-03-07T03:31:15.233571+0000 mgr.y (mgr.34107) 582 : cluster 0 pgmap v1055: 713 pgs: 713 active+clean; 2.6 MiB data, 525 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 5.5 KiB/s wr, 83 op/s 2024-03-07T03:31:17.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:16 smithi145 bash[82329]: cluster 2024-03-07T03:31:15.586844+0000 mon.a (mon.0) 1455 : cluster 0 osdmap e849: 8 total, 8 up, 8 in 2024-03-07T03:31:18.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:17 smithi145 bash[79878]: cluster 2024-03-07T03:31:16.594654+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e850: 8 total, 8 up, 8 in 2024-03-07T03:31:18.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:17 smithi145 bash[79878]: cluster 2024-03-07T03:31:17.556020+0000 mon.a (mon.0) 1457 : cluster 0 osdmap e851: 8 total, 8 up, 8 in 2024-03-07T03:31:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:17 smithi145 bash[82329]: cluster 2024-03-07T03:31:16.594654+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e850: 8 total, 8 up, 8 in 2024-03-07T03:31:18.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:17 smithi145 bash[82329]: cluster 2024-03-07T03:31:17.556020+0000 mon.a (mon.0) 1457 : cluster 0 osdmap e851: 8 total, 8 up, 8 in 2024-03-07T03:31:18.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:17 smithi189 bash[66969]: cluster 2024-03-07T03:31:16.594654+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e850: 8 total, 8 up, 8 in 2024-03-07T03:31:18.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:17 smithi189 bash[66969]: cluster 2024-03-07T03:31:17.556020+0000 mon.a (mon.0) 1457 : cluster 0 osdmap e851: 8 total, 8 up, 8 in 2024-03-07T03:31:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:18 smithi145 bash[79878]: cluster 2024-03-07T03:31:17.236178+0000 mgr.y (mgr.34107) 583 : cluster 0 pgmap v1058: 649 pgs: 649 active+clean; 2.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 13 op/s 2024-03-07T03:31:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:18 smithi145 bash[79878]: audit 2024-03-07T03:31:17.656293+0000 mon.a (mon.0) 1458 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:18 smithi145 bash[79878]: cluster 2024-03-07T03:31:18.559525+0000 mon.a (mon.0) 1459 : cluster 0 osdmap e852: 8 total, 8 up, 8 in 2024-03-07T03:31:19.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:18 smithi145 bash[82329]: cluster 2024-03-07T03:31:17.236178+0000 mgr.y (mgr.34107) 583 : cluster 0 pgmap v1058: 649 pgs: 649 active+clean; 2.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 13 op/s 2024-03-07T03:31:19.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:18 smithi145 bash[82329]: audit 2024-03-07T03:31:17.656293+0000 mon.a (mon.0) 1458 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:19.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:18 smithi145 bash[82329]: cluster 2024-03-07T03:31:18.559525+0000 mon.a (mon.0) 1459 : cluster 0 osdmap e852: 8 total, 8 up, 8 in 2024-03-07T03:31:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:18 smithi189 bash[66969]: cluster 2024-03-07T03:31:17.236178+0000 mgr.y (mgr.34107) 583 : cluster 0 pgmap v1058: 649 pgs: 649 active+clean; 2.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 13 op/s 2024-03-07T03:31:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:18 smithi189 bash[66969]: audit 2024-03-07T03:31:17.656293+0000 mon.a (mon.0) 1458 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:19.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:18 smithi189 bash[66969]: cluster 2024-03-07T03:31:18.559525+0000 mon.a (mon.0) 1459 : cluster 0 osdmap e852: 8 total, 8 up, 8 in 2024-03-07T03:31:20.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:20 smithi145 bash[79878]: cluster 2024-03-07T03:31:19.238708+0000 mgr.y (mgr.34107) 584 : cluster 0 pgmap v1061: 585 pgs: 585 active+clean; 2.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:20.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:20 smithi145 bash[79878]: cluster 2024-03-07T03:31:19.561439+0000 mon.a (mon.0) 1460 : cluster 0 osdmap e853: 8 total, 8 up, 8 in 2024-03-07T03:31:20.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:20 smithi145 bash[82329]: cluster 2024-03-07T03:31:19.238708+0000 mgr.y (mgr.34107) 584 : cluster 0 pgmap v1061: 585 pgs: 585 active+clean; 2.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:20.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:20 smithi145 bash[82329]: cluster 2024-03-07T03:31:19.561439+0000 mon.a (mon.0) 1460 : cluster 0 osdmap e853: 8 total, 8 up, 8 in 2024-03-07T03:31:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:20 smithi189 bash[66969]: cluster 2024-03-07T03:31:19.238708+0000 mgr.y (mgr.34107) 584 : cluster 0 pgmap v1061: 585 pgs: 585 active+clean; 2.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:20 smithi189 bash[66969]: cluster 2024-03-07T03:31:19.561439+0000 mon.a (mon.0) 1460 : cluster 0 osdmap e853: 8 total, 8 up, 8 in 2024-03-07T03:31:21.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:21 smithi189 bash[66969]: cluster 2024-03-07T03:31:20.564618+0000 mon.a (mon.0) 1461 : cluster 0 osdmap e854: 8 total, 8 up, 8 in 2024-03-07T03:31:22.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:21 smithi145 bash[79878]: cluster 2024-03-07T03:31:20.564618+0000 mon.a (mon.0) 1461 : cluster 0 osdmap e854: 8 total, 8 up, 8 in 2024-03-07T03:31:22.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:21 smithi145 bash[82329]: cluster 2024-03-07T03:31:20.564618+0000 mon.a (mon.0) 1461 : cluster 0 osdmap e854: 8 total, 8 up, 8 in 2024-03-07T03:31:22.576 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:31:22 smithi189 bash[59814]: ts=2024-03-07T03:31:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:31:22.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:22 smithi189 bash[66969]: cluster 2024-03-07T03:31:21.240792+0000 mgr.y (mgr.34107) 585 : cluster 0 pgmap v1064: 521 pgs: 521 active+clean; 2.6 MiB data, 534 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:22.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:22 smithi189 bash[66969]: cluster 2024-03-07T03:31:21.593085+0000 mon.a (mon.0) 1462 : cluster 0 osdmap e855: 8 total, 8 up, 8 in 2024-03-07T03:31:22.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:22 smithi189 bash[66969]: cluster 2024-03-07T03:31:22.565658+0000 mon.a (mon.0) 1463 : cluster 0 osdmap e856: 8 total, 8 up, 8 in 2024-03-07T03:31:23.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:31:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:31:22] "GET /metrics HTTP/1.1" 200 75007 "" "Prometheus/2.43.0" 2024-03-07T03:31:23.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:22 smithi145 bash[79878]: cluster 2024-03-07T03:31:21.240792+0000 mgr.y (mgr.34107) 585 : cluster 0 pgmap v1064: 521 pgs: 521 active+clean; 2.6 MiB data, 534 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:23.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:22 smithi145 bash[79878]: cluster 2024-03-07T03:31:21.593085+0000 mon.a (mon.0) 1462 : cluster 0 osdmap e855: 8 total, 8 up, 8 in 2024-03-07T03:31:23.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:22 smithi145 bash[79878]: cluster 2024-03-07T03:31:22.565658+0000 mon.a (mon.0) 1463 : cluster 0 osdmap e856: 8 total, 8 up, 8 in 2024-03-07T03:31:23.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:22 smithi145 bash[82329]: cluster 2024-03-07T03:31:21.240792+0000 mgr.y (mgr.34107) 585 : cluster 0 pgmap v1064: 521 pgs: 521 active+clean; 2.6 MiB data, 534 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:23.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:22 smithi145 bash[82329]: cluster 2024-03-07T03:31:21.593085+0000 mon.a (mon.0) 1462 : cluster 0 osdmap e855: 8 total, 8 up, 8 in 2024-03-07T03:31:23.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:22 smithi145 bash[82329]: cluster 2024-03-07T03:31:22.565658+0000 mon.a (mon.0) 1463 : cluster 0 osdmap e856: 8 total, 8 up, 8 in 2024-03-07T03:31:24.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:24 smithi189 bash[66969]: cluster 2024-03-07T03:31:23.241765+0000 mgr.y (mgr.34107) 586 : cluster 0 pgmap v1067: 457 pgs: 457 active+clean; 2.6 MiB data, 539 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:24.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:24 smithi189 bash[66969]: cluster 2024-03-07T03:31:23.584598+0000 mon.a (mon.0) 1464 : cluster 0 osdmap e857: 8 total, 8 up, 8 in 2024-03-07T03:31:25.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:24 smithi145 bash[79878]: cluster 2024-03-07T03:31:23.241765+0000 mgr.y (mgr.34107) 586 : cluster 0 pgmap v1067: 457 pgs: 457 active+clean; 2.6 MiB data, 539 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:25.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:24 smithi145 bash[79878]: cluster 2024-03-07T03:31:23.584598+0000 mon.a (mon.0) 1464 : cluster 0 osdmap e857: 8 total, 8 up, 8 in 2024-03-07T03:31:25.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:24 smithi145 bash[82329]: cluster 2024-03-07T03:31:23.241765+0000 mgr.y (mgr.34107) 586 : cluster 0 pgmap v1067: 457 pgs: 457 active+clean; 2.6 MiB data, 539 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:25.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:24 smithi145 bash[82329]: cluster 2024-03-07T03:31:23.584598+0000 mon.a (mon.0) 1464 : cluster 0 osdmap e857: 8 total, 8 up, 8 in 2024-03-07T03:31:26.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:25 smithi145 bash[79878]: cluster 2024-03-07T03:31:24.600458+0000 mon.a (mon.0) 1465 : cluster 0 osdmap e858: 8 total, 8 up, 8 in 2024-03-07T03:31:26.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:25 smithi145 bash[82329]: cluster 2024-03-07T03:31:24.600458+0000 mon.a (mon.0) 1465 : cluster 0 osdmap e858: 8 total, 8 up, 8 in 2024-03-07T03:31:26.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:25 smithi189 bash[66969]: cluster 2024-03-07T03:31:24.600458+0000 mon.a (mon.0) 1465 : cluster 0 osdmap e858: 8 total, 8 up, 8 in 2024-03-07T03:31:27.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:26 smithi145 bash[79878]: cluster 2024-03-07T03:31:25.243151+0000 mgr.y (mgr.34107) 587 : cluster 0 pgmap v1070: 393 pgs: 393 active+clean; 2.6 MiB data, 543 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:27.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:26 smithi145 bash[79878]: cluster 2024-03-07T03:31:25.614855+0000 mon.a (mon.0) 1466 : cluster 0 osdmap e859: 8 total, 8 up, 8 in 2024-03-07T03:31:27.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:26 smithi145 bash[82329]: cluster 2024-03-07T03:31:25.243151+0000 mgr.y (mgr.34107) 587 : cluster 0 pgmap v1070: 393 pgs: 393 active+clean; 2.6 MiB data, 543 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:27.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:26 smithi145 bash[82329]: cluster 2024-03-07T03:31:25.614855+0000 mon.a (mon.0) 1466 : cluster 0 osdmap e859: 8 total, 8 up, 8 in 2024-03-07T03:31:27.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:26 smithi189 bash[66969]: cluster 2024-03-07T03:31:25.243151+0000 mgr.y (mgr.34107) 587 : cluster 0 pgmap v1070: 393 pgs: 393 active+clean; 2.6 MiB data, 543 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:27.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:26 smithi189 bash[66969]: cluster 2024-03-07T03:31:25.614855+0000 mon.a (mon.0) 1466 : cluster 0 osdmap e859: 8 total, 8 up, 8 in 2024-03-07T03:31:28.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:27 smithi145 bash[79878]: cluster 2024-03-07T03:31:26.614986+0000 mon.a (mon.0) 1467 : cluster 0 osdmap e860: 8 total, 8 up, 8 in 2024-03-07T03:31:28.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:27 smithi145 bash[82329]: cluster 2024-03-07T03:31:26.614986+0000 mon.a (mon.0) 1467 : cluster 0 osdmap e860: 8 total, 8 up, 8 in 2024-03-07T03:31:28.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:27 smithi189 bash[66969]: cluster 2024-03-07T03:31:26.614986+0000 mon.a (mon.0) 1467 : cluster 0 osdmap e860: 8 total, 8 up, 8 in 2024-03-07T03:31:29.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:28 smithi145 bash[79878]: cluster 2024-03-07T03:31:27.244931+0000 mgr.y (mgr.34107) 588 : cluster 0 pgmap v1073: 329 pgs: 329 active+clean; 587 KiB data, 543 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:29.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:28 smithi145 bash[79878]: cluster 2024-03-07T03:31:27.640850+0000 mon.a (mon.0) 1468 : cluster 0 osdmap e861: 8 total, 8 up, 8 in 2024-03-07T03:31:29.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:28 smithi145 bash[82329]: cluster 2024-03-07T03:31:27.244931+0000 mgr.y (mgr.34107) 588 : cluster 0 pgmap v1073: 329 pgs: 329 active+clean; 587 KiB data, 543 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:29.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:28 smithi145 bash[82329]: cluster 2024-03-07T03:31:27.640850+0000 mon.a (mon.0) 1468 : cluster 0 osdmap e861: 8 total, 8 up, 8 in 2024-03-07T03:31:29.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:28 smithi189 bash[66969]: cluster 2024-03-07T03:31:27.244931+0000 mgr.y (mgr.34107) 588 : cluster 0 pgmap v1073: 329 pgs: 329 active+clean; 587 KiB data, 543 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:29.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:28 smithi189 bash[66969]: cluster 2024-03-07T03:31:27.640850+0000 mon.a (mon.0) 1468 : cluster 0 osdmap e861: 8 total, 8 up, 8 in 2024-03-07T03:31:30.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:29 smithi145 bash[79878]: cluster 2024-03-07T03:31:28.642128+0000 mon.a (mon.0) 1469 : cluster 0 osdmap e862: 8 total, 8 up, 8 in 2024-03-07T03:31:30.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:29 smithi145 bash[82329]: cluster 2024-03-07T03:31:28.642128+0000 mon.a (mon.0) 1469 : cluster 0 osdmap e862: 8 total, 8 up, 8 in 2024-03-07T03:31:30.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:29 smithi189 bash[66969]: cluster 2024-03-07T03:31:28.642128+0000 mon.a (mon.0) 1469 : cluster 0 osdmap e862: 8 total, 8 up, 8 in 2024-03-07T03:31:31.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:30 smithi145 bash[79878]: cluster 2024-03-07T03:31:29.246281+0000 mgr.y (mgr.34107) 589 : cluster 0 pgmap v1076: 265 pgs: 265 active+clean; 587 KiB data, 544 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:31.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:30 smithi145 bash[79878]: cluster 2024-03-07T03:31:29.667961+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e863: 8 total, 8 up, 8 in 2024-03-07T03:31:31.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:30 smithi145 bash[82329]: cluster 2024-03-07T03:31:29.246281+0000 mgr.y (mgr.34107) 589 : cluster 0 pgmap v1076: 265 pgs: 265 active+clean; 587 KiB data, 544 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:31.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:30 smithi145 bash[82329]: cluster 2024-03-07T03:31:29.667961+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e863: 8 total, 8 up, 8 in 2024-03-07T03:31:31.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:30 smithi189 bash[66969]: cluster 2024-03-07T03:31:29.246281+0000 mgr.y (mgr.34107) 589 : cluster 0 pgmap v1076: 265 pgs: 265 active+clean; 587 KiB data, 544 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:31.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:30 smithi189 bash[66969]: cluster 2024-03-07T03:31:29.667961+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e863: 8 total, 8 up, 8 in 2024-03-07T03:31:32.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:31 smithi145 bash[79878]: cluster 2024-03-07T03:31:30.686038+0000 mon.a (mon.0) 1471 : cluster 0 osdmap e864: 8 total, 8 up, 8 in 2024-03-07T03:31:32.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:31 smithi145 bash[82329]: cluster 2024-03-07T03:31:30.686038+0000 mon.a (mon.0) 1471 : cluster 0 osdmap e864: 8 total, 8 up, 8 in 2024-03-07T03:31:32.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:31 smithi189 bash[66969]: cluster 2024-03-07T03:31:30.686038+0000 mon.a (mon.0) 1471 : cluster 0 osdmap e864: 8 total, 8 up, 8 in 2024-03-07T03:31:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:31:32 smithi189 bash[59814]: ts=2024-03-07T03:31:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:31:33.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:31:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:31:32] "GET /metrics HTTP/1.1" 200 40032 "" "Prometheus/2.43.0" 2024-03-07T03:31:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:32 smithi145 bash[79878]: cluster 2024-03-07T03:31:31.247389+0000 mgr.y (mgr.34107) 590 : cluster 0 pgmap v1079: 201 pgs: 201 active+clean; 587 KiB data, 544 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:32 smithi145 bash[79878]: cluster 2024-03-07T03:31:31.687382+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e865: 8 total, 8 up, 8 in 2024-03-07T03:31:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:32 smithi145 bash[79878]: audit 2024-03-07T03:31:32.655924+0000 mon.a (mon.0) 1473 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:32 smithi145 bash[82329]: cluster 2024-03-07T03:31:31.247389+0000 mgr.y (mgr.34107) 590 : cluster 0 pgmap v1079: 201 pgs: 201 active+clean; 587 KiB data, 544 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:32 smithi145 bash[82329]: cluster 2024-03-07T03:31:31.687382+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e865: 8 total, 8 up, 8 in 2024-03-07T03:31:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:32 smithi145 bash[82329]: audit 2024-03-07T03:31:32.655924+0000 mon.a (mon.0) 1473 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:33.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:32 smithi189 bash[66969]: cluster 2024-03-07T03:31:31.247389+0000 mgr.y (mgr.34107) 590 : cluster 0 pgmap v1079: 201 pgs: 201 active+clean; 587 KiB data, 544 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:33.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:32 smithi189 bash[66969]: cluster 2024-03-07T03:31:31.687382+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e865: 8 total, 8 up, 8 in 2024-03-07T03:31:33.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:32 smithi189 bash[66969]: audit 2024-03-07T03:31:32.655924+0000 mon.a (mon.0) 1473 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:34.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:33 smithi145 bash[79878]: cluster 2024-03-07T03:31:32.689439+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e866: 8 total, 8 up, 8 in 2024-03-07T03:31:34.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:33 smithi145 bash[82329]: cluster 2024-03-07T03:31:32.689439+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e866: 8 total, 8 up, 8 in 2024-03-07T03:31:34.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:33 smithi189 bash[66969]: cluster 2024-03-07T03:31:32.689439+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e866: 8 total, 8 up, 8 in 2024-03-07T03:31:34.722 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 168 tests from TestLibRBD (643768 ms total) 2024-03-07T03:31:34.722 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:31:34.722 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/0, where TypeParam = PlaintextUnderLUKS1 2024-03-07T03:31:34.773 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/0.Simple 2024-03-07T03:31:35.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:34 smithi145 bash[79878]: cluster 2024-03-07T03:31:33.248205+0000 mgr.y (mgr.34107) 591 : cluster 0 pgmap v1082: 137 pgs: 137 active+clean; 586 KiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:35.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:34 smithi145 bash[79878]: cluster 2024-03-07T03:31:33.718927+0000 mon.a (mon.0) 1475 : cluster 0 osdmap e867: 8 total, 8 up, 8 in 2024-03-07T03:31:35.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:34 smithi145 bash[82329]: cluster 2024-03-07T03:31:33.248205+0000 mgr.y (mgr.34107) 591 : cluster 0 pgmap v1082: 137 pgs: 137 active+clean; 586 KiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:35.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:34 smithi145 bash[82329]: cluster 2024-03-07T03:31:33.718927+0000 mon.a (mon.0) 1475 : cluster 0 osdmap e867: 8 total, 8 up, 8 in 2024-03-07T03:31:35.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:34 smithi189 bash[66969]: cluster 2024-03-07T03:31:33.248205+0000 mgr.y (mgr.34107) 591 : cluster 0 pgmap v1082: 137 pgs: 137 active+clean; 586 KiB data, 528 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:35.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:34 smithi189 bash[66969]: cluster 2024-03-07T03:31:33.718927+0000 mon.a (mon.0) 1475 : cluster 0 osdmap e867: 8 total, 8 up, 8 in 2024-03-07T03:31:36.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:35 smithi145 bash[79878]: cluster 2024-03-07T03:31:34.730559+0000 mon.a (mon.0) 1476 : cluster 0 osdmap e868: 8 total, 8 up, 8 in 2024-03-07T03:31:36.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:35 smithi145 bash[82329]: cluster 2024-03-07T03:31:34.730559+0000 mon.a (mon.0) 1476 : cluster 0 osdmap e868: 8 total, 8 up, 8 in 2024-03-07T03:31:36.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:35 smithi189 bash[66969]: cluster 2024-03-07T03:31:34.730559+0000 mon.a (mon.0) 1476 : cluster 0 osdmap e868: 8 total, 8 up, 8 in 2024-03-07T03:31:36.744 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:31:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:36 smithi145 bash[79878]: cluster 2024-03-07T03:31:35.249049+0000 mgr.y (mgr.34107) 592 : cluster 0 pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 532 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:36 smithi145 bash[79878]: cluster 2024-03-07T03:31:35.738520+0000 mon.a (mon.0) 1477 : cluster 0 osdmap e869: 8 total, 8 up, 8 in 2024-03-07T03:31:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:36 smithi145 bash[79878]: audit 2024-03-07T03:31:35.742444+0000 mon.c (mon.2) 46 : audit 1 from='client.? 172.21.15.145:0/51922655' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:36 smithi145 bash[79878]: audit 2024-03-07T03:31:35.743537+0000 mon.a (mon.0) 1478 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:36 smithi145 bash[82329]: cluster 2024-03-07T03:31:35.249049+0000 mgr.y (mgr.34107) 592 : cluster 0 pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 532 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:36 smithi145 bash[82329]: cluster 2024-03-07T03:31:35.738520+0000 mon.a (mon.0) 1477 : cluster 0 osdmap e869: 8 total, 8 up, 8 in 2024-03-07T03:31:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:36 smithi145 bash[82329]: audit 2024-03-07T03:31:35.742444+0000 mon.c (mon.2) 46 : audit 1 from='client.? 172.21.15.145:0/51922655' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:37.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:36 smithi145 bash[82329]: audit 2024-03-07T03:31:35.743537+0000 mon.a (mon.0) 1478 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:36 smithi189 bash[66969]: cluster 2024-03-07T03:31:35.249049+0000 mgr.y (mgr.34107) 592 : cluster 0 pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 532 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:36 smithi189 bash[66969]: cluster 2024-03-07T03:31:35.738520+0000 mon.a (mon.0) 1477 : cluster 0 osdmap e869: 8 total, 8 up, 8 in 2024-03-07T03:31:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:36 smithi189 bash[66969]: audit 2024-03-07T03:31:35.742444+0000 mon.c (mon.2) 46 : audit 1 from='client.? 172.21.15.145:0/51922655' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:36 smithi189 bash[66969]: audit 2024-03-07T03:31:35.743537+0000 mon.a (mon.0) 1478 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:38.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:37 smithi145 bash[79878]: audit 2024-03-07T03:31:36.734372+0000 mon.a (mon.0) 1479 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:38.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:37 smithi145 bash[79878]: cluster 2024-03-07T03:31:36.741420+0000 mon.a (mon.0) 1480 : cluster 0 osdmap e870: 8 total, 8 up, 8 in 2024-03-07T03:31:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:37 smithi145 bash[82329]: audit 2024-03-07T03:31:36.734372+0000 mon.a (mon.0) 1479 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:38.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:37 smithi145 bash[82329]: cluster 2024-03-07T03:31:36.741420+0000 mon.a (mon.0) 1480 : cluster 0 osdmap e870: 8 total, 8 up, 8 in 2024-03-07T03:31:38.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:37 smithi189 bash[66969]: audit 2024-03-07T03:31:36.734372+0000 mon.a (mon.0) 1479 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:38.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:37 smithi189 bash[66969]: cluster 2024-03-07T03:31:36.741420+0000 mon.a (mon.0) 1480 : cluster 0 osdmap e870: 8 total, 8 up, 8 in 2024-03-07T03:31:39.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:38 smithi145 bash[79878]: cluster 2024-03-07T03:31:37.249888+0000 mgr.y (mgr.34107) 593 : cluster 0 pgmap v1088: 105 pgs: 9 creating+peering, 15 unknown, 81 active+clean; 582 KiB data, 532 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:38 smithi145 bash[79878]: cluster 2024-03-07T03:31:37.767669+0000 mon.a (mon.0) 1481 : cluster 0 osdmap e871: 8 total, 8 up, 8 in 2024-03-07T03:31:39.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:38 smithi145 bash[82329]: cluster 2024-03-07T03:31:37.249888+0000 mgr.y (mgr.34107) 593 : cluster 0 pgmap v1088: 105 pgs: 9 creating+peering, 15 unknown, 81 active+clean; 582 KiB data, 532 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:39.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:38 smithi145 bash[82329]: cluster 2024-03-07T03:31:37.767669+0000 mon.a (mon.0) 1481 : cluster 0 osdmap e871: 8 total, 8 up, 8 in 2024-03-07T03:31:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:38 smithi189 bash[66969]: cluster 2024-03-07T03:31:37.249888+0000 mgr.y (mgr.34107) 593 : cluster 0 pgmap v1088: 105 pgs: 9 creating+peering, 15 unknown, 81 active+clean; 582 KiB data, 532 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:31:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:38 smithi189 bash[66969]: cluster 2024-03-07T03:31:37.767669+0000 mon.a (mon.0) 1481 : cluster 0 osdmap e871: 8 total, 8 up, 8 in 2024-03-07T03:31:40.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:39 smithi145 bash[82329]: cluster 2024-03-07T03:31:38.766426+0000 mon.a (mon.0) 1482 : cluster 0 osdmap e872: 8 total, 8 up, 8 in 2024-03-07T03:31:40.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:39 smithi145 bash[82329]: audit 2024-03-07T03:31:39.234587+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:31:40.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:39 smithi145 bash[79878]: cluster 2024-03-07T03:31:38.766426+0000 mon.a (mon.0) 1482 : cluster 0 osdmap e872: 8 total, 8 up, 8 in 2024-03-07T03:31:40.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:39 smithi145 bash[79878]: audit 2024-03-07T03:31:39.234587+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:31:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:39 smithi189 bash[66969]: cluster 2024-03-07T03:31:38.766426+0000 mon.a (mon.0) 1482 : cluster 0 osdmap e872: 8 total, 8 up, 8 in 2024-03-07T03:31:40.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:39 smithi189 bash[66969]: audit 2024-03-07T03:31:39.234587+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:31:41.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:40 smithi145 bash[79878]: cluster 2024-03-07T03:31:39.250713+0000 mgr.y (mgr.34107) 594 : cluster 0 pgmap v1091: 105 pgs: 9 creating+peering, 6 unknown, 90 active+clean; 582 KiB data, 533 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.5 KiB/s wr, 14 op/s 2024-03-07T03:31:41.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:40 smithi145 bash[79878]: cluster 2024-03-07T03:31:39.793520+0000 mon.a (mon.0) 1484 : cluster 0 osdmap e873: 8 total, 8 up, 8 in 2024-03-07T03:31:41.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:40 smithi145 bash[82329]: cluster 2024-03-07T03:31:39.250713+0000 mgr.y (mgr.34107) 594 : cluster 0 pgmap v1091: 105 pgs: 9 creating+peering, 6 unknown, 90 active+clean; 582 KiB data, 533 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.5 KiB/s wr, 14 op/s 2024-03-07T03:31:41.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:40 smithi145 bash[82329]: cluster 2024-03-07T03:31:39.793520+0000 mon.a (mon.0) 1484 : cluster 0 osdmap e873: 8 total, 8 up, 8 in 2024-03-07T03:31:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:40 smithi189 bash[66969]: cluster 2024-03-07T03:31:39.250713+0000 mgr.y (mgr.34107) 594 : cluster 0 pgmap v1091: 105 pgs: 9 creating+peering, 6 unknown, 90 active+clean; 582 KiB data, 533 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.5 KiB/s wr, 14 op/s 2024-03-07T03:31:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:40 smithi189 bash[66969]: cluster 2024-03-07T03:31:39.793520+0000 mon.a (mon.0) 1484 : cluster 0 osdmap e873: 8 total, 8 up, 8 in 2024-03-07T03:31:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:31:42 smithi189 bash[59814]: ts=2024-03-07T03:31:42.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:31:43.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:42 smithi145 bash[79878]: cluster 2024-03-07T03:31:41.251650+0000 mgr.y (mgr.34107) 595 : cluster 0 pgmap v1093: 105 pgs: 3 creating+peering, 102 active+clean; 13 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 2.7 MiB/s wr, 57 op/s 2024-03-07T03:31:43.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:31:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:31:42] "GET /metrics HTTP/1.1" 200 40032 "" "Prometheus/2.43.0" 2024-03-07T03:31:43.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:42 smithi145 bash[82329]: cluster 2024-03-07T03:31:41.251650+0000 mgr.y (mgr.34107) 595 : cluster 0 pgmap v1093: 105 pgs: 3 creating+peering, 102 active+clean; 13 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 2.7 MiB/s wr, 57 op/s 2024-03-07T03:31:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:42 smithi189 bash[66969]: cluster 2024-03-07T03:31:41.251650+0000 mgr.y (mgr.34107) 595 : cluster 0 pgmap v1093: 105 pgs: 3 creating+peering, 102 active+clean; 13 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 2.7 MiB/s wr, 57 op/s 2024-03-07T03:31:45.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:44 smithi145 bash[79878]: cluster 2024-03-07T03:31:43.252483+0000 mgr.y (mgr.34107) 596 : cluster 0 pgmap v1094: 105 pgs: 105 active+clean; 17 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-03-07T03:31:45.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:44 smithi145 bash[82329]: cluster 2024-03-07T03:31:43.252483+0000 mgr.y (mgr.34107) 596 : cluster 0 pgmap v1094: 105 pgs: 105 active+clean; 17 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-03-07T03:31:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:44 smithi189 bash[66969]: cluster 2024-03-07T03:31:43.252483+0000 mgr.y (mgr.34107) 596 : cluster 0 pgmap v1094: 105 pgs: 105 active+clean; 17 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-03-07T03:31:46.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[79878]: audit 2024-03-07T03:31:45.028461+0000 mon.a (mon.0) 1485 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:46.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[79878]: audit 2024-03-07T03:31:45.039420+0000 mon.a (mon.0) 1486 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:46.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[79878]: cluster 2024-03-07T03:31:45.253198+0000 mgr.y (mgr.34107) 597 : cluster 0 pgmap v1095: 105 pgs: 105 active+clean; 26 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 3.3 MiB/s wr, 54 op/s 2024-03-07T03:31:46.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[79878]: audit 2024-03-07T03:31:45.378419+0000 mon.a (mon.0) 1487 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:46.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[79878]: audit 2024-03-07T03:31:45.388848+0000 mon.a (mon.0) 1488 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:46.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[79878]: audit 2024-03-07T03:31:46.016277+0000 mon.a (mon.0) 1489 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:31:46.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[79878]: audit 2024-03-07T03:31:46.018429+0000 mon.a (mon.0) 1490 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:31:46.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[82329]: audit 2024-03-07T03:31:45.028461+0000 mon.a (mon.0) 1485 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:46.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[82329]: audit 2024-03-07T03:31:45.039420+0000 mon.a (mon.0) 1486 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:46.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[82329]: cluster 2024-03-07T03:31:45.253198+0000 mgr.y (mgr.34107) 597 : cluster 0 pgmap v1095: 105 pgs: 105 active+clean; 26 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 3.3 MiB/s wr, 54 op/s 2024-03-07T03:31:46.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[82329]: audit 2024-03-07T03:31:45.378419+0000 mon.a (mon.0) 1487 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:46.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[82329]: audit 2024-03-07T03:31:45.388848+0000 mon.a (mon.0) 1488 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:46.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[82329]: audit 2024-03-07T03:31:46.016277+0000 mon.a (mon.0) 1489 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:31:46.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:46 smithi145 bash[82329]: audit 2024-03-07T03:31:46.018429+0000 mon.a (mon.0) 1490 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:31:46.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:46 smithi189 bash[66969]: audit 2024-03-07T03:31:45.028461+0000 mon.a (mon.0) 1485 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:46.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:46 smithi189 bash[66969]: audit 2024-03-07T03:31:45.039420+0000 mon.a (mon.0) 1486 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:46.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:46 smithi189 bash[66969]: cluster 2024-03-07T03:31:45.253198+0000 mgr.y (mgr.34107) 597 : cluster 0 pgmap v1095: 105 pgs: 105 active+clean; 26 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 3.3 MiB/s wr, 54 op/s 2024-03-07T03:31:46.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:46 smithi189 bash[66969]: audit 2024-03-07T03:31:45.378419+0000 mon.a (mon.0) 1487 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:46.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:46 smithi189 bash[66969]: audit 2024-03-07T03:31:45.388848+0000 mon.a (mon.0) 1488 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:46.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:46 smithi189 bash[66969]: audit 2024-03-07T03:31:46.016277+0000 mon.a (mon.0) 1489 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:31:46.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:46 smithi189 bash[66969]: audit 2024-03-07T03:31:46.018429+0000 mon.a (mon.0) 1490 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:31:47.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:47 smithi145 bash[82329]: audit 2024-03-07T03:31:46.044565+0000 mon.a (mon.0) 1491 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:47.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:47 smithi145 bash[79878]: audit 2024-03-07T03:31:46.044565+0000 mon.a (mon.0) 1491 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:47.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:47 smithi189 bash[66969]: audit 2024-03-07T03:31:46.044565+0000 mon.a (mon.0) 1491 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:48.576 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/0.Simple (13802 ms) 2024-03-07T03:31:48.576 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/0.Grow 2024-03-07T03:31:48.576 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:31:48.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:48 smithi145 bash[79878]: cluster 2024-03-07T03:31:47.253766+0000 mgr.y (mgr.34107) 598 : cluster 0 pgmap v1096: 105 pgs: 105 active+clean; 39 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 5.0 MiB/s wr, 60 op/s 2024-03-07T03:31:48.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:48 smithi145 bash[79878]: cluster 2024-03-07T03:31:47.571021+0000 mon.a (mon.0) 1492 : cluster 0 osdmap e874: 8 total, 8 up, 8 in 2024-03-07T03:31:48.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:48 smithi145 bash[79878]: audit 2024-03-07T03:31:47.656010+0000 mon.a (mon.0) 1493 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:48.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:48 smithi145 bash[82329]: cluster 2024-03-07T03:31:47.253766+0000 mgr.y (mgr.34107) 598 : cluster 0 pgmap v1096: 105 pgs: 105 active+clean; 39 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 5.0 MiB/s wr, 60 op/s 2024-03-07T03:31:48.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:48 smithi145 bash[82329]: cluster 2024-03-07T03:31:47.571021+0000 mon.a (mon.0) 1492 : cluster 0 osdmap e874: 8 total, 8 up, 8 in 2024-03-07T03:31:48.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:48 smithi145 bash[82329]: audit 2024-03-07T03:31:47.656010+0000 mon.a (mon.0) 1493 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:48.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:48 smithi189 bash[66969]: cluster 2024-03-07T03:31:47.253766+0000 mgr.y (mgr.34107) 598 : cluster 0 pgmap v1096: 105 pgs: 105 active+clean; 39 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 5.0 MiB/s wr, 60 op/s 2024-03-07T03:31:48.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:48 smithi189 bash[66969]: cluster 2024-03-07T03:31:47.571021+0000 mon.a (mon.0) 1492 : cluster 0 osdmap e874: 8 total, 8 up, 8 in 2024-03-07T03:31:48.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:48 smithi189 bash[66969]: audit 2024-03-07T03:31:47.656010+0000 mon.a (mon.0) 1493 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:50.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:50 smithi145 bash[79878]: cluster 2024-03-07T03:31:49.254641+0000 mgr.y (mgr.34107) 599 : cluster 0 pgmap v1098: 105 pgs: 105 active+clean; 43 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 4.9 MiB/s wr, 58 op/s 2024-03-07T03:31:50.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:50 smithi145 bash[79878]: audit 2024-03-07T03:31:49.464050+0000 mon.a (mon.0) 1494 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:50.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:50 smithi145 bash[79878]: cluster 2024-03-07T03:31:49.584902+0000 mon.a (mon.0) 1495 : cluster 0 osdmap e875: 8 total, 8 up, 8 in 2024-03-07T03:31:50.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:50 smithi145 bash[82329]: cluster 2024-03-07T03:31:49.254641+0000 mgr.y (mgr.34107) 599 : cluster 0 pgmap v1098: 105 pgs: 105 active+clean; 43 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 4.9 MiB/s wr, 58 op/s 2024-03-07T03:31:50.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:50 smithi145 bash[82329]: audit 2024-03-07T03:31:49.464050+0000 mon.a (mon.0) 1494 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:50.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:50 smithi145 bash[82329]: cluster 2024-03-07T03:31:49.584902+0000 mon.a (mon.0) 1495 : cluster 0 osdmap e875: 8 total, 8 up, 8 in 2024-03-07T03:31:50.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:50 smithi189 bash[66969]: cluster 2024-03-07T03:31:49.254641+0000 mgr.y (mgr.34107) 599 : cluster 0 pgmap v1098: 105 pgs: 105 active+clean; 43 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 4.9 MiB/s wr, 58 op/s 2024-03-07T03:31:50.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:50 smithi189 bash[66969]: audit 2024-03-07T03:31:49.464050+0000 mon.a (mon.0) 1494 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:31:50.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:50 smithi189 bash[66969]: cluster 2024-03-07T03:31:49.584902+0000 mon.a (mon.0) 1495 : cluster 0 osdmap e875: 8 total, 8 up, 8 in 2024-03-07T03:31:52.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:52 smithi189 bash[66969]: cluster 2024-03-07T03:31:51.255573+0000 mgr.y (mgr.34107) 600 : cluster 0 pgmap v1100: 105 pgs: 105 active+clean; 63 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 6.3 MiB/s wr, 61 op/s 2024-03-07T03:31:52.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:31:52 smithi189 bash[59814]: ts=2024-03-07T03:31:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:31:52.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:52 smithi145 bash[79878]: cluster 2024-03-07T03:31:51.255573+0000 mgr.y (mgr.34107) 600 : cluster 0 pgmap v1100: 105 pgs: 105 active+clean; 63 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 6.3 MiB/s wr, 61 op/s 2024-03-07T03:31:52.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:52 smithi145 bash[82329]: cluster 2024-03-07T03:31:51.255573+0000 mgr.y (mgr.34107) 600 : cluster 0 pgmap v1100: 105 pgs: 105 active+clean; 63 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 6.3 MiB/s wr, 61 op/s 2024-03-07T03:31:53.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:31:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:31:52] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.43.0" 2024-03-07T03:31:54.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:54 smithi145 bash[79878]: cluster 2024-03-07T03:31:53.256512+0000 mgr.y (mgr.34107) 601 : cluster 0 pgmap v1101: 105 pgs: 105 active+clean; 76 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 6.8 MiB/s wr, 72 op/s 2024-03-07T03:31:54.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:54 smithi145 bash[82329]: cluster 2024-03-07T03:31:53.256512+0000 mgr.y (mgr.34107) 601 : cluster 0 pgmap v1101: 105 pgs: 105 active+clean; 76 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 6.8 MiB/s wr, 72 op/s 2024-03-07T03:31:54.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:54 smithi189 bash[66969]: cluster 2024-03-07T03:31:53.256512+0000 mgr.y (mgr.34107) 601 : cluster 0 pgmap v1101: 105 pgs: 105 active+clean; 76 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 6.8 MiB/s wr, 72 op/s 2024-03-07T03:31:56.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:56 smithi145 bash[79878]: cluster 2024-03-07T03:31:55.257140+0000 mgr.y (mgr.34107) 602 : cluster 0 pgmap v1102: 105 pgs: 105 active+clean; 84 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.6 MiB/s wr, 63 op/s 2024-03-07T03:31:56.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:56 smithi145 bash[82329]: cluster 2024-03-07T03:31:55.257140+0000 mgr.y (mgr.34107) 602 : cluster 0 pgmap v1102: 105 pgs: 105 active+clean; 84 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.6 MiB/s wr, 63 op/s 2024-03-07T03:31:56.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:56 smithi189 bash[66969]: cluster 2024-03-07T03:31:55.257140+0000 mgr.y (mgr.34107) 602 : cluster 0 pgmap v1102: 105 pgs: 105 active+clean; 84 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.6 MiB/s wr, 63 op/s 2024-03-07T03:31:57.982 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/0.Grow (9406 ms) 2024-03-07T03:31:57.982 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/0.Shrink 2024-03-07T03:31:57.982 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:31:58.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:58 smithi145 bash[79878]: cluster 2024-03-07T03:31:57.257659+0000 mgr.y (mgr.34107) 603 : cluster 0 pgmap v1103: 105 pgs: 105 active+clean; 96 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 5.9 MiB/s wr, 70 op/s 2024-03-07T03:31:58.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:58 smithi145 bash[82329]: cluster 2024-03-07T03:31:57.257659+0000 mgr.y (mgr.34107) 603 : cluster 0 pgmap v1103: 105 pgs: 105 active+clean; 96 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 5.9 MiB/s wr, 70 op/s 2024-03-07T03:31:58.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:58 smithi189 bash[66969]: cluster 2024-03-07T03:31:57.257659+0000 mgr.y (mgr.34107) 603 : cluster 0 pgmap v1103: 105 pgs: 105 active+clean; 96 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 5.9 MiB/s wr, 70 op/s 2024-03-07T03:31:59.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:31:59 smithi145 bash[82329]: cluster 2024-03-07T03:31:58.484026+0000 mon.a (mon.0) 1496 : cluster 0 osdmap e876: 8 total, 8 up, 8 in 2024-03-07T03:31:59.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:31:59 smithi145 bash[79878]: cluster 2024-03-07T03:31:58.484026+0000 mon.a (mon.0) 1496 : cluster 0 osdmap e876: 8 total, 8 up, 8 in 2024-03-07T03:31:59.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:31:59 smithi189 bash[66969]: cluster 2024-03-07T03:31:58.484026+0000 mon.a (mon.0) 1496 : cluster 0 osdmap e876: 8 total, 8 up, 8 in 2024-03-07T03:32:00.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:00 smithi145 bash[79878]: cluster 2024-03-07T03:31:59.258608+0000 mgr.y (mgr.34107) 604 : cluster 0 pgmap v1105: 105 pgs: 105 active+clean; 117 MiB data, 905 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 8.1 MiB/s wr, 67 op/s 2024-03-07T03:32:00.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:00 smithi145 bash[82329]: cluster 2024-03-07T03:31:59.258608+0000 mgr.y (mgr.34107) 604 : cluster 0 pgmap v1105: 105 pgs: 105 active+clean; 117 MiB data, 905 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 8.1 MiB/s wr, 67 op/s 2024-03-07T03:32:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:00 smithi189 bash[66969]: cluster 2024-03-07T03:31:59.258608+0000 mgr.y (mgr.34107) 604 : cluster 0 pgmap v1105: 105 pgs: 105 active+clean; 117 MiB data, 905 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 8.1 MiB/s wr, 67 op/s 2024-03-07T03:32:02.580 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:02 smithi189 bash[66969]: cluster 2024-03-07T03:32:01.259539+0000 mgr.y (mgr.34107) 605 : cluster 0 pgmap v1106: 105 pgs: 105 active+clean; 130 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.1 MiB/s wr, 57 op/s 2024-03-07T03:32:02.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:32:02 smithi189 bash[59814]: ts=2024-03-07T03:32:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:32:02.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:02 smithi145 bash[79878]: cluster 2024-03-07T03:32:01.259539+0000 mgr.y (mgr.34107) 605 : cluster 0 pgmap v1106: 105 pgs: 105 active+clean; 130 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.1 MiB/s wr, 57 op/s 2024-03-07T03:32:02.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:02 smithi145 bash[82329]: cluster 2024-03-07T03:32:01.259539+0000 mgr.y (mgr.34107) 605 : cluster 0 pgmap v1106: 105 pgs: 105 active+clean; 130 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.1 MiB/s wr, 57 op/s 2024-03-07T03:32:03.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:32:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:32:02] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2024-03-07T03:32:03.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:03 smithi145 bash[82329]: audit 2024-03-07T03:32:02.656433+0000 mon.a (mon.0) 1497 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:03.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:03 smithi145 bash[79878]: audit 2024-03-07T03:32:02.656433+0000 mon.a (mon.0) 1497 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:03.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:03 smithi189 bash[66969]: audit 2024-03-07T03:32:02.656433+0000 mon.a (mon.0) 1497 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:04.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:04 smithi145 bash[79878]: cluster 2024-03-07T03:32:03.260381+0000 mgr.y (mgr.34107) 606 : cluster 0 pgmap v1107: 105 pgs: 105 active+clean; 134 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 63 op/s 2024-03-07T03:32:04.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:04 smithi145 bash[82329]: cluster 2024-03-07T03:32:03.260381+0000 mgr.y (mgr.34107) 606 : cluster 0 pgmap v1107: 105 pgs: 105 active+clean; 134 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 63 op/s 2024-03-07T03:32:04.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:04 smithi189 bash[66969]: cluster 2024-03-07T03:32:03.260381+0000 mgr.y (mgr.34107) 606 : cluster 0 pgmap v1107: 105 pgs: 105 active+clean; 134 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 63 op/s 2024-03-07T03:32:06.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:06 smithi145 bash[79878]: cluster 2024-03-07T03:32:05.261369+0000 mgr.y (mgr.34107) 607 : cluster 0 pgmap v1108: 105 pgs: 105 active+clean; 138 MiB data, 971 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.2 MiB/s wr, 67 op/s 2024-03-07T03:32:06.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:06 smithi145 bash[82329]: cluster 2024-03-07T03:32:05.261369+0000 mgr.y (mgr.34107) 607 : cluster 0 pgmap v1108: 105 pgs: 105 active+clean; 138 MiB data, 971 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.2 MiB/s wr, 67 op/s 2024-03-07T03:32:06.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:06 smithi189 bash[66969]: cluster 2024-03-07T03:32:05.261369+0000 mgr.y (mgr.34107) 607 : cluster 0 pgmap v1108: 105 pgs: 105 active+clean; 138 MiB data, 971 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.2 MiB/s wr, 67 op/s 2024-03-07T03:32:06.938 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/0.Shrink (8957 ms) 2024-03-07T03:32:06.939 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/0.ShrinkToOne 2024-03-07T03:32:06.939 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:32:08.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:08 smithi145 bash[79878]: cluster 2024-03-07T03:32:07.262355+0000 mgr.y (mgr.34107) 608 : cluster 0 pgmap v1109: 105 pgs: 105 active+clean; 155 MiB data, 1022 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.7 MiB/s wr, 66 op/s 2024-03-07T03:32:08.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:08 smithi145 bash[79878]: cluster 2024-03-07T03:32:07.536974+0000 mon.a (mon.0) 1498 : cluster 0 osdmap e877: 8 total, 8 up, 8 in 2024-03-07T03:32:08.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:08 smithi145 bash[82329]: cluster 2024-03-07T03:32:07.262355+0000 mgr.y (mgr.34107) 608 : cluster 0 pgmap v1109: 105 pgs: 105 active+clean; 155 MiB data, 1022 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.7 MiB/s wr, 66 op/s 2024-03-07T03:32:08.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:08 smithi145 bash[82329]: cluster 2024-03-07T03:32:07.536974+0000 mon.a (mon.0) 1498 : cluster 0 osdmap e877: 8 total, 8 up, 8 in 2024-03-07T03:32:08.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:08 smithi189 bash[66969]: cluster 2024-03-07T03:32:07.262355+0000 mgr.y (mgr.34107) 608 : cluster 0 pgmap v1109: 105 pgs: 105 active+clean; 155 MiB data, 1022 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.7 MiB/s wr, 66 op/s 2024-03-07T03:32:08.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:08 smithi189 bash[66969]: cluster 2024-03-07T03:32:07.536974+0000 mon.a (mon.0) 1498 : cluster 0 osdmap e877: 8 total, 8 up, 8 in 2024-03-07T03:32:10.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:10 smithi145 bash[79878]: cluster 2024-03-07T03:32:09.263184+0000 mgr.y (mgr.34107) 609 : cluster 0 pgmap v1111: 105 pgs: 105 active+clean; 168 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.5 MiB/s wr, 69 op/s 2024-03-07T03:32:10.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:10 smithi145 bash[82329]: cluster 2024-03-07T03:32:09.263184+0000 mgr.y (mgr.34107) 609 : cluster 0 pgmap v1111: 105 pgs: 105 active+clean; 168 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.5 MiB/s wr, 69 op/s 2024-03-07T03:32:10.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:10 smithi189 bash[66969]: cluster 2024-03-07T03:32:09.263184+0000 mgr.y (mgr.34107) 609 : cluster 0 pgmap v1111: 105 pgs: 105 active+clean; 168 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.5 MiB/s wr, 69 op/s 2024-03-07T03:32:12.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:12 smithi189 bash[66969]: cluster 2024-03-07T03:32:11.264072+0000 mgr.y (mgr.34107) 610 : cluster 0 pgmap v1112: 105 pgs: 105 active+clean; 176 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.0 MiB/s wr, 89 op/s 2024-03-07T03:32:12.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:32:12 smithi189 bash[59814]: ts=2024-03-07T03:32:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:32:12.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:12 smithi145 bash[79878]: cluster 2024-03-07T03:32:11.264072+0000 mgr.y (mgr.34107) 610 : cluster 0 pgmap v1112: 105 pgs: 105 active+clean; 176 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.0 MiB/s wr, 89 op/s 2024-03-07T03:32:12.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:32:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:32:12] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2024-03-07T03:32:12.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:12 smithi145 bash[82329]: cluster 2024-03-07T03:32:11.264072+0000 mgr.y (mgr.34107) 610 : cluster 0 pgmap v1112: 105 pgs: 105 active+clean; 176 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.0 MiB/s wr, 89 op/s 2024-03-07T03:32:14.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:14 smithi145 bash[82329]: cluster 2024-03-07T03:32:13.264930+0000 mgr.y (mgr.34107) 611 : cluster 0 pgmap v1113: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 72 op/s 2024-03-07T03:32:14.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:14 smithi145 bash[79878]: cluster 2024-03-07T03:32:13.264930+0000 mgr.y (mgr.34107) 611 : cluster 0 pgmap v1113: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 72 op/s 2024-03-07T03:32:14.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:14 smithi189 bash[66969]: cluster 2024-03-07T03:32:13.264930+0000 mgr.y (mgr.34107) 611 : cluster 0 pgmap v1113: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 72 op/s 2024-03-07T03:32:15.959 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/0.ShrinkToOne (9020 ms) 2024-03-07T03:32:15.959 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/0.ShrinkToOneAfterSnapshot 2024-03-07T03:32:15.959 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:32:16.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:16 smithi145 bash[82329]: cluster 2024-03-07T03:32:15.265896+0000 mgr.y (mgr.34107) 612 : cluster 0 pgmap v1114: 105 pgs: 105 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.8 MiB/s wr, 71 op/s 2024-03-07T03:32:16.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:16 smithi145 bash[79878]: cluster 2024-03-07T03:32:15.265896+0000 mgr.y (mgr.34107) 612 : cluster 0 pgmap v1114: 105 pgs: 105 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.8 MiB/s wr, 71 op/s 2024-03-07T03:32:16.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:16 smithi189 bash[66969]: cluster 2024-03-07T03:32:15.265896+0000 mgr.y (mgr.34107) 612 : cluster 0 pgmap v1114: 105 pgs: 105 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.8 MiB/s wr, 71 op/s 2024-03-07T03:32:17.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:17 smithi189 bash[66969]: cluster 2024-03-07T03:32:16.559365+0000 mon.a (mon.0) 1499 : cluster 0 osdmap e878: 8 total, 8 up, 8 in 2024-03-07T03:32:18.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:17 smithi145 bash[79878]: cluster 2024-03-07T03:32:16.559365+0000 mon.a (mon.0) 1499 : cluster 0 osdmap e878: 8 total, 8 up, 8 in 2024-03-07T03:32:18.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:17 smithi145 bash[82329]: cluster 2024-03-07T03:32:16.559365+0000 mon.a (mon.0) 1499 : cluster 0 osdmap e878: 8 total, 8 up, 8 in 2024-03-07T03:32:18.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:18 smithi189 bash[66969]: cluster 2024-03-07T03:32:17.266750+0000 mgr.y (mgr.34107) 613 : cluster 0 pgmap v1116: 105 pgs: 105 active+clean; 204 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 5.9 MiB/s wr, 74 op/s 2024-03-07T03:32:18.830 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:18 smithi189 bash[66969]: audit 2024-03-07T03:32:17.656844+0000 mon.a (mon.0) 1500 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:19.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:18 smithi145 bash[82329]: cluster 2024-03-07T03:32:17.266750+0000 mgr.y (mgr.34107) 613 : cluster 0 pgmap v1116: 105 pgs: 105 active+clean; 204 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 5.9 MiB/s wr, 74 op/s 2024-03-07T03:32:19.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:18 smithi145 bash[82329]: audit 2024-03-07T03:32:17.656844+0000 mon.a (mon.0) 1500 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:18 smithi145 bash[79878]: cluster 2024-03-07T03:32:17.266750+0000 mgr.y (mgr.34107) 613 : cluster 0 pgmap v1116: 105 pgs: 105 active+clean; 204 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 5.9 MiB/s wr, 74 op/s 2024-03-07T03:32:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:18 smithi145 bash[79878]: audit 2024-03-07T03:32:17.656844+0000 mon.a (mon.0) 1500 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:20 smithi189 bash[66969]: cluster 2024-03-07T03:32:19.267598+0000 mgr.y (mgr.34107) 614 : cluster 0 pgmap v1117: 105 pgs: 105 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 5.6 MiB/s wr, 71 op/s 2024-03-07T03:32:21.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:20 smithi145 bash[82329]: cluster 2024-03-07T03:32:19.267598+0000 mgr.y (mgr.34107) 614 : cluster 0 pgmap v1117: 105 pgs: 105 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 5.6 MiB/s wr, 71 op/s 2024-03-07T03:32:21.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:20 smithi145 bash[79878]: cluster 2024-03-07T03:32:19.267598+0000 mgr.y (mgr.34107) 614 : cluster 0 pgmap v1117: 105 pgs: 105 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 5.6 MiB/s wr, 71 op/s 2024-03-07T03:32:22.569 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:32:22 smithi189 bash[59814]: ts=2024-03-07T03:32:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:32:22.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:22 smithi189 bash[66969]: cluster 2024-03-07T03:32:21.268537+0000 mgr.y (mgr.34107) 615 : cluster 0 pgmap v1118: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 952 KiB/s rd, 5.3 MiB/s wr, 64 op/s 2024-03-07T03:32:23.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:22 smithi145 bash[79878]: cluster 2024-03-07T03:32:21.268537+0000 mgr.y (mgr.34107) 615 : cluster 0 pgmap v1118: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 952 KiB/s rd, 5.3 MiB/s wr, 64 op/s 2024-03-07T03:32:23.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:32:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:32:22] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2024-03-07T03:32:23.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:22 smithi145 bash[82329]: cluster 2024-03-07T03:32:21.268537+0000 mgr.y (mgr.34107) 615 : cluster 0 pgmap v1118: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 952 KiB/s rd, 5.3 MiB/s wr, 64 op/s 2024-03-07T03:32:25.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:24 smithi145 bash[79878]: cluster 2024-03-07T03:32:23.269464+0000 mgr.y (mgr.34107) 616 : cluster 0 pgmap v1119: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 946 KiB/s rd, 4.1 MiB/s wr, 55 op/s 2024-03-07T03:32:25.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:24 smithi145 bash[79878]: cluster 2024-03-07T03:32:23.595748+0000 mon.a (mon.0) 1501 : cluster 0 osdmap e879: 8 total, 8 up, 8 in 2024-03-07T03:32:25.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:24 smithi145 bash[82329]: cluster 2024-03-07T03:32:23.269464+0000 mgr.y (mgr.34107) 616 : cluster 0 pgmap v1119: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 946 KiB/s rd, 4.1 MiB/s wr, 55 op/s 2024-03-07T03:32:25.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:24 smithi145 bash[82329]: cluster 2024-03-07T03:32:23.595748+0000 mon.a (mon.0) 1501 : cluster 0 osdmap e879: 8 total, 8 up, 8 in 2024-03-07T03:32:25.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:24 smithi189 bash[66969]: cluster 2024-03-07T03:32:23.269464+0000 mgr.y (mgr.34107) 616 : cluster 0 pgmap v1119: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 946 KiB/s rd, 4.1 MiB/s wr, 55 op/s 2024-03-07T03:32:25.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:24 smithi189 bash[66969]: cluster 2024-03-07T03:32:23.595748+0000 mon.a (mon.0) 1501 : cluster 0 osdmap e879: 8 total, 8 up, 8 in 2024-03-07T03:32:26.064 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/0.ShrinkToOneAfterSnapshot (10106 ms) 2024-03-07T03:32:26.065 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/0.MinOverlap 2024-03-07T03:32:26.065 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:32:27.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:26 smithi145 bash[79878]: cluster 2024-03-07T03:32:25.270474+0000 mgr.y (mgr.34107) 617 : cluster 0 pgmap v1121: 105 pgs: 105 active+clean; 246 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 5.7 MiB/s wr, 72 op/s 2024-03-07T03:32:27.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:26 smithi145 bash[82329]: cluster 2024-03-07T03:32:25.270474+0000 mgr.y (mgr.34107) 617 : cluster 0 pgmap v1121: 105 pgs: 105 active+clean; 246 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 5.7 MiB/s wr, 72 op/s 2024-03-07T03:32:27.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:26 smithi189 bash[66969]: cluster 2024-03-07T03:32:25.270474+0000 mgr.y (mgr.34107) 617 : cluster 0 pgmap v1121: 105 pgs: 105 active+clean; 246 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 5.7 MiB/s wr, 72 op/s 2024-03-07T03:32:28.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:27 smithi145 bash[82329]: cluster 2024-03-07T03:32:26.610617+0000 mon.a (mon.0) 1502 : cluster 0 osdmap e880: 8 total, 8 up, 8 in 2024-03-07T03:32:28.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:27 smithi145 bash[79878]: cluster 2024-03-07T03:32:26.610617+0000 mon.a (mon.0) 1502 : cluster 0 osdmap e880: 8 total, 8 up, 8 in 2024-03-07T03:32:28.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:27 smithi189 bash[66969]: cluster 2024-03-07T03:32:26.610617+0000 mon.a (mon.0) 1502 : cluster 0 osdmap e880: 8 total, 8 up, 8 in 2024-03-07T03:32:29.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:28 smithi145 bash[82329]: cluster 2024-03-07T03:32:27.271301+0000 mgr.y (mgr.34107) 618 : cluster 0 pgmap v1123: 105 pgs: 105 active+clean; 278 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 7.8 MiB/s wr, 97 op/s 2024-03-07T03:32:29.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:28 smithi145 bash[79878]: cluster 2024-03-07T03:32:27.271301+0000 mgr.y (mgr.34107) 618 : cluster 0 pgmap v1123: 105 pgs: 105 active+clean; 278 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 7.8 MiB/s wr, 97 op/s 2024-03-07T03:32:29.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:28 smithi189 bash[66969]: cluster 2024-03-07T03:32:27.271301+0000 mgr.y (mgr.34107) 618 : cluster 0 pgmap v1123: 105 pgs: 105 active+clean; 278 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 7.8 MiB/s wr, 97 op/s 2024-03-07T03:32:31.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:30 smithi145 bash[82329]: cluster 2024-03-07T03:32:29.272265+0000 mgr.y (mgr.34107) 619 : cluster 0 pgmap v1124: 105 pgs: 105 active+clean; 294 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2024-03-07T03:32:31.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:30 smithi145 bash[79878]: cluster 2024-03-07T03:32:29.272265+0000 mgr.y (mgr.34107) 619 : cluster 0 pgmap v1124: 105 pgs: 105 active+clean; 294 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2024-03-07T03:32:31.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:30 smithi189 bash[66969]: cluster 2024-03-07T03:32:29.272265+0000 mgr.y (mgr.34107) 619 : cluster 0 pgmap v1124: 105 pgs: 105 active+clean; 294 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2024-03-07T03:32:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:32:32 smithi189 bash[59814]: ts=2024-03-07T03:32:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:32:33.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:32 smithi145 bash[79878]: cluster 2024-03-07T03:32:31.273189+0000 mgr.y (mgr.34107) 620 : cluster 0 pgmap v1125: 105 pgs: 105 active+clean; 307 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 11 MiB/s wr, 91 op/s 2024-03-07T03:32:33.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:32:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:32:32] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.43.0" 2024-03-07T03:32:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:32 smithi145 bash[82329]: cluster 2024-03-07T03:32:31.273189+0000 mgr.y (mgr.34107) 620 : cluster 0 pgmap v1125: 105 pgs: 105 active+clean; 307 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 11 MiB/s wr, 91 op/s 2024-03-07T03:32:33.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:32 smithi189 bash[66969]: cluster 2024-03-07T03:32:31.273189+0000 mgr.y (mgr.34107) 620 : cluster 0 pgmap v1125: 105 pgs: 105 active+clean; 307 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 11 MiB/s wr, 91 op/s 2024-03-07T03:32:34.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:33 smithi145 bash[79878]: audit 2024-03-07T03:32:32.657146+0000 mon.a (mon.0) 1503 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:34.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:33 smithi145 bash[82329]: audit 2024-03-07T03:32:32.657146+0000 mon.a (mon.0) 1503 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:34.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:33 smithi189 bash[66969]: audit 2024-03-07T03:32:32.657146+0000 mon.a (mon.0) 1503 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:35.054 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/0.MinOverlap (8989 ms) 2024-03-07T03:32:35.055 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/0.ZeroOverlap 2024-03-07T03:32:35.055 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:32:35.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:34 smithi145 bash[82329]: cluster 2024-03-07T03:32:33.274031+0000 mgr.y (mgr.34107) 621 : cluster 0 pgmap v1126: 105 pgs: 105 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 8.9 MiB/s wr, 79 op/s 2024-03-07T03:32:35.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:34 smithi145 bash[79878]: cluster 2024-03-07T03:32:33.274031+0000 mgr.y (mgr.34107) 621 : cluster 0 pgmap v1126: 105 pgs: 105 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 8.9 MiB/s wr, 79 op/s 2024-03-07T03:32:35.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:34 smithi189 bash[66969]: cluster 2024-03-07T03:32:33.274031+0000 mgr.y (mgr.34107) 621 : cluster 0 pgmap v1126: 105 pgs: 105 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 8.9 MiB/s wr, 79 op/s 2024-03-07T03:32:37.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:36 smithi145 bash[79878]: cluster 2024-03-07T03:32:35.275093+0000 mgr.y (mgr.34107) 622 : cluster 0 pgmap v1127: 105 pgs: 105 active+clean; 315 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 7.7 MiB/s wr, 79 op/s 2024-03-07T03:32:37.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:36 smithi145 bash[79878]: cluster 2024-03-07T03:32:35.648701+0000 mon.a (mon.0) 1504 : cluster 0 osdmap e881: 8 total, 8 up, 8 in 2024-03-07T03:32:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:36 smithi145 bash[82329]: cluster 2024-03-07T03:32:35.275093+0000 mgr.y (mgr.34107) 622 : cluster 0 pgmap v1127: 105 pgs: 105 active+clean; 315 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 7.7 MiB/s wr, 79 op/s 2024-03-07T03:32:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:36 smithi145 bash[82329]: cluster 2024-03-07T03:32:35.648701+0000 mon.a (mon.0) 1504 : cluster 0 osdmap e881: 8 total, 8 up, 8 in 2024-03-07T03:32:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:36 smithi189 bash[66969]: cluster 2024-03-07T03:32:35.275093+0000 mgr.y (mgr.34107) 622 : cluster 0 pgmap v1127: 105 pgs: 105 active+clean; 315 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 7.7 MiB/s wr, 79 op/s 2024-03-07T03:32:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:36 smithi189 bash[66969]: cluster 2024-03-07T03:32:35.648701+0000 mon.a (mon.0) 1504 : cluster 0 osdmap e881: 8 total, 8 up, 8 in 2024-03-07T03:32:39.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:38 smithi145 bash[79878]: cluster 2024-03-07T03:32:37.275763+0000 mgr.y (mgr.34107) 623 : cluster 0 pgmap v1129: 105 pgs: 105 active+clean; 336 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 6.6 MiB/s wr, 66 op/s 2024-03-07T03:32:39.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:38 smithi145 bash[82329]: cluster 2024-03-07T03:32:37.275763+0000 mgr.y (mgr.34107) 623 : cluster 0 pgmap v1129: 105 pgs: 105 active+clean; 336 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 6.6 MiB/s wr, 66 op/s 2024-03-07T03:32:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:38 smithi189 bash[66969]: cluster 2024-03-07T03:32:37.275763+0000 mgr.y (mgr.34107) 623 : cluster 0 pgmap v1129: 105 pgs: 105 active+clean; 336 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 6.6 MiB/s wr, 66 op/s 2024-03-07T03:32:41.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:40 smithi145 bash[79878]: cluster 2024-03-07T03:32:39.276266+0000 mgr.y (mgr.34107) 624 : cluster 0 pgmap v1130: 105 pgs: 105 active+clean; 340 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.4 MiB/s wr, 65 op/s 2024-03-07T03:32:41.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:40 smithi145 bash[82329]: cluster 2024-03-07T03:32:39.276266+0000 mgr.y (mgr.34107) 624 : cluster 0 pgmap v1130: 105 pgs: 105 active+clean; 340 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.4 MiB/s wr, 65 op/s 2024-03-07T03:32:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:40 smithi189 bash[66969]: cluster 2024-03-07T03:32:39.276266+0000 mgr.y (mgr.34107) 624 : cluster 0 pgmap v1130: 105 pgs: 105 active+clean; 340 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.4 MiB/s wr, 65 op/s 2024-03-07T03:32:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:32:42 smithi189 bash[59814]: ts=2024-03-07T03:32:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:32:43.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:32:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:32:42] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.43.0" 2024-03-07T03:32:43.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:42 smithi145 bash[82329]: cluster 2024-03-07T03:32:41.276795+0000 mgr.y (mgr.34107) 625 : cluster 0 pgmap v1131: 105 pgs: 105 active+clean; 340 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 63 op/s 2024-03-07T03:32:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:42 smithi145 bash[79878]: cluster 2024-03-07T03:32:41.276795+0000 mgr.y (mgr.34107) 625 : cluster 0 pgmap v1131: 105 pgs: 105 active+clean; 340 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 63 op/s 2024-03-07T03:32:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:42 smithi189 bash[66969]: cluster 2024-03-07T03:32:41.276795+0000 mgr.y (mgr.34107) 625 : cluster 0 pgmap v1131: 105 pgs: 105 active+clean; 340 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 63 op/s 2024-03-07T03:32:43.979 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/0.ZeroOverlap (8925 ms) 2024-03-07T03:32:44.668 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/0 (69205 ms total) 2024-03-07T03:32:44.668 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:32:44.668 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/1, where TypeParam = PlaintextUnderLUKS2 2024-03-07T03:32:44.726 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/1.Simple 2024-03-07T03:32:45.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:44 smithi145 bash[79878]: cluster 2024-03-07T03:32:43.277289+0000 mgr.y (mgr.34107) 626 : cluster 0 pgmap v1132: 105 pgs: 105 active+clean; 344 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.9 MiB/s wr, 63 op/s 2024-03-07T03:32:45.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:44 smithi145 bash[82329]: cluster 2024-03-07T03:32:43.277289+0000 mgr.y (mgr.34107) 626 : cluster 0 pgmap v1132: 105 pgs: 105 active+clean; 344 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.9 MiB/s wr, 63 op/s 2024-03-07T03:32:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:44 smithi189 bash[66969]: cluster 2024-03-07T03:32:43.277289+0000 mgr.y (mgr.34107) 626 : cluster 0 pgmap v1132: 105 pgs: 105 active+clean; 344 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.9 MiB/s wr, 63 op/s 2024-03-07T03:32:46.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:45 smithi145 bash[79878]: cluster 2024-03-07T03:32:44.671081+0000 mon.a (mon.0) 1505 : cluster 0 osdmap e882: 8 total, 8 up, 8 in 2024-03-07T03:32:46.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:45 smithi145 bash[82329]: cluster 2024-03-07T03:32:44.671081+0000 mon.a (mon.0) 1505 : cluster 0 osdmap e882: 8 total, 8 up, 8 in 2024-03-07T03:32:46.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:45 smithi189 bash[66969]: cluster 2024-03-07T03:32:44.671081+0000 mon.a (mon.0) 1505 : cluster 0 osdmap e882: 8 total, 8 up, 8 in 2024-03-07T03:32:46.711 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:32:47.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:46 smithi145 bash[79878]: cluster 2024-03-07T03:32:45.277914+0000 mgr.y (mgr.34107) 627 : cluster 0 pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:32:47.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:46 smithi145 bash[79878]: cluster 2024-03-07T03:32:45.702711+0000 mon.a (mon.0) 1506 : cluster 0 osdmap e883: 8 total, 8 up, 8 in 2024-03-07T03:32:47.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:46 smithi145 bash[79878]: audit 2024-03-07T03:32:45.705545+0000 mon.a (mon.0) 1507 : audit 1 from='client.? 172.21.15.145:0/915587947' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:32:47.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:46 smithi145 bash[79878]: audit 2024-03-07T03:32:46.125710+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:32:47.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:46 smithi145 bash[82329]: cluster 2024-03-07T03:32:45.277914+0000 mgr.y (mgr.34107) 627 : cluster 0 pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:32:47.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:46 smithi145 bash[82329]: cluster 2024-03-07T03:32:45.702711+0000 mon.a (mon.0) 1506 : cluster 0 osdmap e883: 8 total, 8 up, 8 in 2024-03-07T03:32:47.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:46 smithi145 bash[82329]: audit 2024-03-07T03:32:45.705545+0000 mon.a (mon.0) 1507 : audit 1 from='client.? 172.21.15.145:0/915587947' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:32:47.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:46 smithi145 bash[82329]: audit 2024-03-07T03:32:46.125710+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:32:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:46 smithi189 bash[66969]: cluster 2024-03-07T03:32:45.277914+0000 mgr.y (mgr.34107) 627 : cluster 0 pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:32:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:46 smithi189 bash[66969]: cluster 2024-03-07T03:32:45.702711+0000 mon.a (mon.0) 1506 : cluster 0 osdmap e883: 8 total, 8 up, 8 in 2024-03-07T03:32:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:46 smithi189 bash[66969]: audit 2024-03-07T03:32:45.705545+0000 mon.a (mon.0) 1507 : audit 1 from='client.? 172.21.15.145:0/915587947' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:32:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:46 smithi189 bash[66969]: audit 2024-03-07T03:32:46.125710+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:32:48.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:47 smithi145 bash[79878]: audit 2024-03-07T03:32:46.696835+0000 mon.a (mon.0) 1509 : audit 1 from='client.? 172.21.15.145:0/915587947' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:32:48.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:47 smithi145 bash[79878]: cluster 2024-03-07T03:32:46.709001+0000 mon.a (mon.0) 1510 : cluster 0 osdmap e884: 8 total, 8 up, 8 in 2024-03-07T03:32:48.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:47 smithi145 bash[79878]: audit 2024-03-07T03:32:47.658271+0000 mon.a (mon.0) 1511 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:48.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:47 smithi145 bash[82329]: audit 2024-03-07T03:32:46.696835+0000 mon.a (mon.0) 1509 : audit 1 from='client.? 172.21.15.145:0/915587947' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:32:48.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:47 smithi145 bash[82329]: cluster 2024-03-07T03:32:46.709001+0000 mon.a (mon.0) 1510 : cluster 0 osdmap e884: 8 total, 8 up, 8 in 2024-03-07T03:32:48.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:47 smithi145 bash[82329]: audit 2024-03-07T03:32:47.658271+0000 mon.a (mon.0) 1511 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:47 smithi189 bash[66969]: audit 2024-03-07T03:32:46.696835+0000 mon.a (mon.0) 1509 : audit 1 from='client.? 172.21.15.145:0/915587947' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:32:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:47 smithi189 bash[66969]: cluster 2024-03-07T03:32:46.709001+0000 mon.a (mon.0) 1510 : cluster 0 osdmap e884: 8 total, 8 up, 8 in 2024-03-07T03:32:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:47 smithi189 bash[66969]: audit 2024-03-07T03:32:47.658271+0000 mon.a (mon.0) 1511 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:49.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:48 smithi145 bash[79878]: cluster 2024-03-07T03:32:47.278646+0000 mgr.y (mgr.34107) 628 : cluster 0 pgmap v1137: 105 pgs: 8 creating+peering, 19 unknown, 78 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:32:49.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:48 smithi145 bash[79878]: cluster 2024-03-07T03:32:47.735000+0000 mon.a (mon.0) 1512 : cluster 0 osdmap e885: 8 total, 8 up, 8 in 2024-03-07T03:32:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:48 smithi145 bash[82329]: cluster 2024-03-07T03:32:47.278646+0000 mgr.y (mgr.34107) 628 : cluster 0 pgmap v1137: 105 pgs: 8 creating+peering, 19 unknown, 78 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:32:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:48 smithi145 bash[82329]: cluster 2024-03-07T03:32:47.735000+0000 mon.a (mon.0) 1512 : cluster 0 osdmap e885: 8 total, 8 up, 8 in 2024-03-07T03:32:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:48 smithi189 bash[66969]: cluster 2024-03-07T03:32:47.278646+0000 mgr.y (mgr.34107) 628 : cluster 0 pgmap v1137: 105 pgs: 8 creating+peering, 19 unknown, 78 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:32:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:48 smithi189 bash[66969]: cluster 2024-03-07T03:32:47.735000+0000 mon.a (mon.0) 1512 : cluster 0 osdmap e885: 8 total, 8 up, 8 in 2024-03-07T03:32:50.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:49 smithi145 bash[79878]: cluster 2024-03-07T03:32:48.734909+0000 mon.a (mon.0) 1513 : cluster 0 osdmap e886: 8 total, 8 up, 8 in 2024-03-07T03:32:50.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:49 smithi145 bash[82329]: cluster 2024-03-07T03:32:48.734909+0000 mon.a (mon.0) 1513 : cluster 0 osdmap e886: 8 total, 8 up, 8 in 2024-03-07T03:32:50.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:49 smithi189 bash[66969]: cluster 2024-03-07T03:32:48.734909+0000 mon.a (mon.0) 1513 : cluster 0 osdmap e886: 8 total, 8 up, 8 in 2024-03-07T03:32:51.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:50 smithi145 bash[79878]: cluster 2024-03-07T03:32:49.279525+0000 mgr.y (mgr.34107) 629 : cluster 0 pgmap v1140: 105 pgs: 8 creating+peering, 7 unknown, 90 active+clean; 4.6 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 1024 KiB/s wr, 0 op/s 2024-03-07T03:32:51.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:50 smithi145 bash[79878]: cluster 2024-03-07T03:32:49.755671+0000 mon.a (mon.0) 1514 : cluster 0 osdmap e887: 8 total, 8 up, 8 in 2024-03-07T03:32:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:50 smithi145 bash[82329]: cluster 2024-03-07T03:32:49.279525+0000 mgr.y (mgr.34107) 629 : cluster 0 pgmap v1140: 105 pgs: 8 creating+peering, 7 unknown, 90 active+clean; 4.6 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 1024 KiB/s wr, 0 op/s 2024-03-07T03:32:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:50 smithi145 bash[82329]: cluster 2024-03-07T03:32:49.755671+0000 mon.a (mon.0) 1514 : cluster 0 osdmap e887: 8 total, 8 up, 8 in 2024-03-07T03:32:51.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:50 smithi189 bash[66969]: cluster 2024-03-07T03:32:49.279525+0000 mgr.y (mgr.34107) 629 : cluster 0 pgmap v1140: 105 pgs: 8 creating+peering, 7 unknown, 90 active+clean; 4.6 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 1024 KiB/s wr, 0 op/s 2024-03-07T03:32:51.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:50 smithi189 bash[66969]: cluster 2024-03-07T03:32:49.755671+0000 mon.a (mon.0) 1514 : cluster 0 osdmap e887: 8 total, 8 up, 8 in 2024-03-07T03:32:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:32:52 smithi189 bash[59814]: ts=2024-03-07T03:32:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:32:53.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:32:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:32:52] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.43.0" 2024-03-07T03:32:53.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:52 smithi145 bash[79878]: cluster 2024-03-07T03:32:51.280139+0000 mgr.y (mgr.34107) 630 : cluster 0 pgmap v1142: 105 pgs: 3 creating+peering, 102 active+clean; 22 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 4.6 MiB/s wr, 67 op/s 2024-03-07T03:32:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:52 smithi145 bash[79878]: audit 2024-03-07T03:32:52.118710+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:52 smithi145 bash[79878]: audit 2024-03-07T03:32:52.124653+0000 mon.a (mon.0) 1516 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:52 smithi145 bash[79878]: audit 2024-03-07T03:32:52.228890+0000 mon.a (mon.0) 1517 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:52 smithi145 bash[79878]: audit 2024-03-07T03:32:52.237162+0000 mon.a (mon.0) 1518 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:52 smithi145 bash[82329]: cluster 2024-03-07T03:32:51.280139+0000 mgr.y (mgr.34107) 630 : cluster 0 pgmap v1142: 105 pgs: 3 creating+peering, 102 active+clean; 22 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 4.6 MiB/s wr, 67 op/s 2024-03-07T03:32:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:52 smithi145 bash[82329]: audit 2024-03-07T03:32:52.118710+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:52 smithi145 bash[82329]: audit 2024-03-07T03:32:52.124653+0000 mon.a (mon.0) 1516 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:52 smithi145 bash[82329]: audit 2024-03-07T03:32:52.228890+0000 mon.a (mon.0) 1517 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:53.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:52 smithi145 bash[82329]: audit 2024-03-07T03:32:52.237162+0000 mon.a (mon.0) 1518 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:53.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:52 smithi189 bash[66969]: cluster 2024-03-07T03:32:51.280139+0000 mgr.y (mgr.34107) 630 : cluster 0 pgmap v1142: 105 pgs: 3 creating+peering, 102 active+clean; 22 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 4.6 MiB/s wr, 67 op/s 2024-03-07T03:32:53.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:52 smithi189 bash[66969]: audit 2024-03-07T03:32:52.118710+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:53.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:52 smithi189 bash[66969]: audit 2024-03-07T03:32:52.124653+0000 mon.a (mon.0) 1516 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:53.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:52 smithi189 bash[66969]: audit 2024-03-07T03:32:52.228890+0000 mon.a (mon.0) 1517 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:53.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:52 smithi189 bash[66969]: audit 2024-03-07T03:32:52.237162+0000 mon.a (mon.0) 1518 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:54.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:53 smithi145 bash[79878]: audit 2024-03-07T03:32:52.867769+0000 mon.a (mon.0) 1519 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:32:54.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:53 smithi145 bash[79878]: audit 2024-03-07T03:32:52.869007+0000 mon.a (mon.0) 1520 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:32:54.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:53 smithi145 bash[79878]: audit 2024-03-07T03:32:52.876938+0000 mon.a (mon.0) 1521 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:53 smithi145 bash[82329]: audit 2024-03-07T03:32:52.867769+0000 mon.a (mon.0) 1519 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:32:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:53 smithi145 bash[82329]: audit 2024-03-07T03:32:52.869007+0000 mon.a (mon.0) 1520 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:32:54.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:53 smithi145 bash[82329]: audit 2024-03-07T03:32:52.876938+0000 mon.a (mon.0) 1521 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:54.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:53 smithi189 bash[66969]: audit 2024-03-07T03:32:52.867769+0000 mon.a (mon.0) 1519 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:32:54.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:53 smithi189 bash[66969]: audit 2024-03-07T03:32:52.869007+0000 mon.a (mon.0) 1520 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:32:54.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:53 smithi189 bash[66969]: audit 2024-03-07T03:32:52.876938+0000 mon.a (mon.0) 1521 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:32:55.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:54 smithi145 bash[79878]: cluster 2024-03-07T03:32:53.280754+0000 mgr.y (mgr.34107) 631 : cluster 0 pgmap v1143: 105 pgs: 105 active+clean; 26 MiB data, 630 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 4.2 MiB/s wr, 61 op/s 2024-03-07T03:32:55.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:54 smithi145 bash[82329]: cluster 2024-03-07T03:32:53.280754+0000 mgr.y (mgr.34107) 631 : cluster 0 pgmap v1143: 105 pgs: 105 active+clean; 26 MiB data, 630 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 4.2 MiB/s wr, 61 op/s 2024-03-07T03:32:55.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:54 smithi189 bash[66969]: cluster 2024-03-07T03:32:53.280754+0000 mgr.y (mgr.34107) 631 : cluster 0 pgmap v1143: 105 pgs: 105 active+clean; 26 MiB data, 630 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 4.2 MiB/s wr, 61 op/s 2024-03-07T03:32:57.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:56 smithi145 bash[79878]: cluster 2024-03-07T03:32:55.281826+0000 mgr.y (mgr.34107) 632 : cluster 0 pgmap v1144: 105 pgs: 105 active+clean; 26 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 3.3 MiB/s wr, 49 op/s 2024-03-07T03:32:57.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:56 smithi145 bash[82329]: cluster 2024-03-07T03:32:55.281826+0000 mgr.y (mgr.34107) 632 : cluster 0 pgmap v1144: 105 pgs: 105 active+clean; 26 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 3.3 MiB/s wr, 49 op/s 2024-03-07T03:32:57.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:56 smithi189 bash[66969]: cluster 2024-03-07T03:32:55.281826+0000 mgr.y (mgr.34107) 632 : cluster 0 pgmap v1144: 105 pgs: 105 active+clean; 26 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 3.3 MiB/s wr, 49 op/s 2024-03-07T03:32:58.788 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/1.Simple (14062 ms) 2024-03-07T03:32:58.788 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/1.Grow 2024-03-07T03:32:58.788 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:32:58.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:58 smithi189 bash[66969]: cluster 2024-03-07T03:32:57.282650+0000 mgr.y (mgr.34107) 633 : cluster 0 pgmap v1145: 105 pgs: 105 active+clean; 26 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 513 KiB/s rd, 2.9 MiB/s wr, 47 op/s 2024-03-07T03:32:58.832 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:32:58 smithi189 bash[66969]: cluster 2024-03-07T03:32:57.587511+0000 mon.a (mon.0) 1522 : cluster 0 osdmap e888: 8 total, 8 up, 8 in 2024-03-07T03:32:59.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:58 smithi145 bash[79878]: cluster 2024-03-07T03:32:57.282650+0000 mgr.y (mgr.34107) 633 : cluster 0 pgmap v1145: 105 pgs: 105 active+clean; 26 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 513 KiB/s rd, 2.9 MiB/s wr, 47 op/s 2024-03-07T03:32:59.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:32:58 smithi145 bash[79878]: cluster 2024-03-07T03:32:57.587511+0000 mon.a (mon.0) 1522 : cluster 0 osdmap e888: 8 total, 8 up, 8 in 2024-03-07T03:32:59.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:58 smithi145 bash[82329]: cluster 2024-03-07T03:32:57.282650+0000 mgr.y (mgr.34107) 633 : cluster 0 pgmap v1145: 105 pgs: 105 active+clean; 26 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 513 KiB/s rd, 2.9 MiB/s wr, 47 op/s 2024-03-07T03:32:59.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:32:58 smithi145 bash[82329]: cluster 2024-03-07T03:32:57.587511+0000 mon.a (mon.0) 1522 : cluster 0 osdmap e888: 8 total, 8 up, 8 in 2024-03-07T03:33:00.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:00 smithi145 bash[82329]: cluster 2024-03-07T03:32:59.283611+0000 mgr.y (mgr.34107) 634 : cluster 0 pgmap v1147: 105 pgs: 105 active+clean; 38 MiB data, 720 MiB used, 715 GiB / 715 GiB avail; 3.4 MiB/s rd, 3.9 MiB/s wr, 44 op/s 2024-03-07T03:33:00.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:00 smithi145 bash[82329]: audit 2024-03-07T03:32:59.494326+0000 mon.a (mon.0) 1523 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:33:00.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:00 smithi145 bash[82329]: cluster 2024-03-07T03:32:59.599069+0000 mon.a (mon.0) 1524 : cluster 0 osdmap e889: 8 total, 8 up, 8 in 2024-03-07T03:33:00.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:00 smithi145 bash[79878]: cluster 2024-03-07T03:32:59.283611+0000 mgr.y (mgr.34107) 634 : cluster 0 pgmap v1147: 105 pgs: 105 active+clean; 38 MiB data, 720 MiB used, 715 GiB / 715 GiB avail; 3.4 MiB/s rd, 3.9 MiB/s wr, 44 op/s 2024-03-07T03:33:00.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:00 smithi145 bash[79878]: audit 2024-03-07T03:32:59.494326+0000 mon.a (mon.0) 1523 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:33:00.810 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:00 smithi145 bash[79878]: cluster 2024-03-07T03:32:59.599069+0000 mon.a (mon.0) 1524 : cluster 0 osdmap e889: 8 total, 8 up, 8 in 2024-03-07T03:33:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:00 smithi189 bash[66969]: cluster 2024-03-07T03:32:59.283611+0000 mgr.y (mgr.34107) 634 : cluster 0 pgmap v1147: 105 pgs: 105 active+clean; 38 MiB data, 720 MiB used, 715 GiB / 715 GiB avail; 3.4 MiB/s rd, 3.9 MiB/s wr, 44 op/s 2024-03-07T03:33:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:00 smithi189 bash[66969]: audit 2024-03-07T03:32:59.494326+0000 mon.a (mon.0) 1523 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:33:00.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:00 smithi189 bash[66969]: cluster 2024-03-07T03:32:59.599069+0000 mon.a (mon.0) 1524 : cluster 0 osdmap e889: 8 total, 8 up, 8 in 2024-03-07T03:33:02.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:02 smithi189 bash[66969]: cluster 2024-03-07T03:33:01.284377+0000 mgr.y (mgr.34107) 635 : cluster 0 pgmap v1149: 105 pgs: 105 active+clean; 71 MiB data, 786 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 7.6 MiB/s wr, 63 op/s 2024-03-07T03:33:02.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:33:02 smithi189 bash[59814]: ts=2024-03-07T03:33:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:33:02.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:02 smithi145 bash[79878]: cluster 2024-03-07T03:33:01.284377+0000 mgr.y (mgr.34107) 635 : cluster 0 pgmap v1149: 105 pgs: 105 active+clean; 71 MiB data, 786 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 7.6 MiB/s wr, 63 op/s 2024-03-07T03:33:02.750 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:02 smithi145 bash[82329]: cluster 2024-03-07T03:33:01.284377+0000 mgr.y (mgr.34107) 635 : cluster 0 pgmap v1149: 105 pgs: 105 active+clean; 71 MiB data, 786 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 7.6 MiB/s wr, 63 op/s 2024-03-07T03:33:03.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:33:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:33:02] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.43.0" 2024-03-07T03:33:03.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:03 smithi145 bash[79878]: audit 2024-03-07T03:33:02.658064+0000 mon.a (mon.0) 1525 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:03.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:03 smithi145 bash[82329]: audit 2024-03-07T03:33:02.658064+0000 mon.a (mon.0) 1525 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:03.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:03 smithi189 bash[66969]: audit 2024-03-07T03:33:02.658064+0000 mon.a (mon.0) 1525 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:04.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:04 smithi145 bash[79878]: cluster 2024-03-07T03:33:03.284997+0000 mgr.y (mgr.34107) 636 : cluster 0 pgmap v1150: 105 pgs: 105 active+clean; 84 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 9.3 MiB/s wr, 72 op/s 2024-03-07T03:33:04.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:04 smithi145 bash[82329]: cluster 2024-03-07T03:33:03.284997+0000 mgr.y (mgr.34107) 636 : cluster 0 pgmap v1150: 105 pgs: 105 active+clean; 84 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 9.3 MiB/s wr, 72 op/s 2024-03-07T03:33:04.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:04 smithi189 bash[66969]: cluster 2024-03-07T03:33:03.284997+0000 mgr.y (mgr.34107) 636 : cluster 0 pgmap v1150: 105 pgs: 105 active+clean; 84 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 9.3 MiB/s wr, 72 op/s 2024-03-07T03:33:06.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:06 smithi145 bash[79878]: cluster 2024-03-07T03:33:05.285708+0000 mgr.y (mgr.34107) 637 : cluster 0 pgmap v1151: 105 pgs: 105 active+clean; 92 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 10 MiB/s wr, 78 op/s 2024-03-07T03:33:06.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:06 smithi145 bash[82329]: cluster 2024-03-07T03:33:05.285708+0000 mgr.y (mgr.34107) 637 : cluster 0 pgmap v1151: 105 pgs: 105 active+clean; 92 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 10 MiB/s wr, 78 op/s 2024-03-07T03:33:06.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:06 smithi189 bash[66969]: cluster 2024-03-07T03:33:05.285708+0000 mgr.y (mgr.34107) 637 : cluster 0 pgmap v1151: 105 pgs: 105 active+clean; 92 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 10 MiB/s wr, 78 op/s 2024-03-07T03:33:08.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:08 smithi145 bash[79878]: cluster 2024-03-07T03:33:07.286393+0000 mgr.y (mgr.34107) 638 : cluster 0 pgmap v1152: 105 pgs: 105 active+clean; 92 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.5 MiB/s wr, 65 op/s 2024-03-07T03:33:08.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:08 smithi145 bash[82329]: cluster 2024-03-07T03:33:07.286393+0000 mgr.y (mgr.34107) 638 : cluster 0 pgmap v1152: 105 pgs: 105 active+clean; 92 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.5 MiB/s wr, 65 op/s 2024-03-07T03:33:08.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:08 smithi189 bash[66969]: cluster 2024-03-07T03:33:07.286393+0000 mgr.y (mgr.34107) 638 : cluster 0 pgmap v1152: 105 pgs: 105 active+clean; 92 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.5 MiB/s wr, 65 op/s 2024-03-07T03:33:10.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:10 smithi145 bash[79878]: cluster 2024-03-07T03:33:09.287103+0000 mgr.y (mgr.34107) 639 : cluster 0 pgmap v1153: 105 pgs: 105 active+clean; 92 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 6.6 MiB/s wr, 61 op/s 2024-03-07T03:33:10.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:10 smithi145 bash[82329]: cluster 2024-03-07T03:33:09.287103+0000 mgr.y (mgr.34107) 639 : cluster 0 pgmap v1153: 105 pgs: 105 active+clean; 92 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 6.6 MiB/s wr, 61 op/s 2024-03-07T03:33:10.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:10 smithi189 bash[66969]: cluster 2024-03-07T03:33:09.287103+0000 mgr.y (mgr.34107) 639 : cluster 0 pgmap v1153: 105 pgs: 105 active+clean; 92 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 6.6 MiB/s wr, 61 op/s 2024-03-07T03:33:11.084 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/1.Grow (12296 ms) 2024-03-07T03:33:11.084 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/1.Shrink 2024-03-07T03:33:11.084 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:33:12.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:12 smithi189 bash[66969]: cluster 2024-03-07T03:33:11.288074+0000 mgr.y (mgr.34107) 640 : cluster 0 pgmap v1154: 105 pgs: 105 active+clean; 120 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 9.1 MiB/s wr, 76 op/s 2024-03-07T03:33:12.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:12 smithi189 bash[66969]: cluster 2024-03-07T03:33:11.533924+0000 mon.a (mon.0) 1526 : cluster 0 osdmap e890: 8 total, 8 up, 8 in 2024-03-07T03:33:12.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:33:12 smithi189 bash[59814]: ts=2024-03-07T03:33:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:33:12.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:12 smithi145 bash[79878]: cluster 2024-03-07T03:33:11.288074+0000 mgr.y (mgr.34107) 640 : cluster 0 pgmap v1154: 105 pgs: 105 active+clean; 120 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 9.1 MiB/s wr, 76 op/s 2024-03-07T03:33:12.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:12 smithi145 bash[79878]: cluster 2024-03-07T03:33:11.533924+0000 mon.a (mon.0) 1526 : cluster 0 osdmap e890: 8 total, 8 up, 8 in 2024-03-07T03:33:12.808 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:33:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:33:12] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.43.0" 2024-03-07T03:33:12.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:12 smithi145 bash[82329]: cluster 2024-03-07T03:33:11.288074+0000 mgr.y (mgr.34107) 640 : cluster 0 pgmap v1154: 105 pgs: 105 active+clean; 120 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 9.1 MiB/s wr, 76 op/s 2024-03-07T03:33:12.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:12 smithi145 bash[82329]: cluster 2024-03-07T03:33:11.533924+0000 mon.a (mon.0) 1526 : cluster 0 osdmap e890: 8 total, 8 up, 8 in 2024-03-07T03:33:14.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:14 smithi145 bash[79878]: cluster 2024-03-07T03:33:13.288797+0000 mgr.y (mgr.34107) 641 : cluster 0 pgmap v1156: 105 pgs: 105 active+clean; 132 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 6.4 MiB/s wr, 38 op/s 2024-03-07T03:33:14.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:14 smithi145 bash[82329]: cluster 2024-03-07T03:33:13.288797+0000 mgr.y (mgr.34107) 641 : cluster 0 pgmap v1156: 105 pgs: 105 active+clean; 132 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 6.4 MiB/s wr, 38 op/s 2024-03-07T03:33:14.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:14 smithi189 bash[66969]: cluster 2024-03-07T03:33:13.288797+0000 mgr.y (mgr.34107) 641 : cluster 0 pgmap v1156: 105 pgs: 105 active+clean; 132 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 6.4 MiB/s wr, 38 op/s 2024-03-07T03:33:16.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:16 smithi145 bash[79878]: cluster 2024-03-07T03:33:15.289902+0000 mgr.y (mgr.34107) 642 : cluster 0 pgmap v1157: 105 pgs: 105 active+clean; 145 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.9 MiB/s wr, 67 op/s 2024-03-07T03:33:16.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:16 smithi145 bash[82329]: cluster 2024-03-07T03:33:15.289902+0000 mgr.y (mgr.34107) 642 : cluster 0 pgmap v1157: 105 pgs: 105 active+clean; 145 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.9 MiB/s wr, 67 op/s 2024-03-07T03:33:16.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:16 smithi189 bash[66969]: cluster 2024-03-07T03:33:15.289902+0000 mgr.y (mgr.34107) 642 : cluster 0 pgmap v1157: 105 pgs: 105 active+clean; 145 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.9 MiB/s wr, 67 op/s 2024-03-07T03:33:18.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:18 smithi145 bash[82329]: cluster 2024-03-07T03:33:17.290532+0000 mgr.y (mgr.34107) 643 : cluster 0 pgmap v1158: 105 pgs: 105 active+clean; 158 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-03-07T03:33:18.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:18 smithi145 bash[82329]: audit 2024-03-07T03:33:17.658156+0000 mon.a (mon.0) 1527 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:18.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:18 smithi145 bash[79878]: cluster 2024-03-07T03:33:17.290532+0000 mgr.y (mgr.34107) 643 : cluster 0 pgmap v1158: 105 pgs: 105 active+clean; 158 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-03-07T03:33:18.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:18 smithi145 bash[79878]: audit 2024-03-07T03:33:17.658156+0000 mon.a (mon.0) 1527 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:18.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:18 smithi189 bash[66969]: cluster 2024-03-07T03:33:17.290532+0000 mgr.y (mgr.34107) 643 : cluster 0 pgmap v1158: 105 pgs: 105 active+clean; 158 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-03-07T03:33:18.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:18 smithi189 bash[66969]: audit 2024-03-07T03:33:17.658156+0000 mon.a (mon.0) 1527 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:20.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:20 smithi145 bash[82329]: cluster 2024-03-07T03:33:19.291163+0000 mgr.y (mgr.34107) 644 : cluster 0 pgmap v1159: 105 pgs: 105 active+clean; 162 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 68 op/s 2024-03-07T03:33:20.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:20 smithi145 bash[79878]: cluster 2024-03-07T03:33:19.291163+0000 mgr.y (mgr.34107) 644 : cluster 0 pgmap v1159: 105 pgs: 105 active+clean; 162 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 68 op/s 2024-03-07T03:33:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:20 smithi189 bash[66969]: cluster 2024-03-07T03:33:19.291163+0000 mgr.y (mgr.34107) 644 : cluster 0 pgmap v1159: 105 pgs: 105 active+clean; 162 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 68 op/s 2024-03-07T03:33:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:22 smithi189 bash[66969]: cluster 2024-03-07T03:33:21.291797+0000 mgr.y (mgr.34107) 645 : cluster 0 pgmap v1160: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 5.4 MiB/s wr, 45 op/s 2024-03-07T03:33:22.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:33:22 smithi189 bash[59814]: ts=2024-03-07T03:33:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:33:22.792 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/1.Shrink (11705 ms) 2024-03-07T03:33:22.792 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/1.ShrinkToOne 2024-03-07T03:33:22.792 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:33:22.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:22 smithi145 bash[79878]: cluster 2024-03-07T03:33:21.291797+0000 mgr.y (mgr.34107) 645 : cluster 0 pgmap v1160: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 5.4 MiB/s wr, 45 op/s 2024-03-07T03:33:22.808 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:33:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:33:22] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2024-03-07T03:33:22.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:22 smithi145 bash[82329]: cluster 2024-03-07T03:33:21.291797+0000 mgr.y (mgr.34107) 645 : cluster 0 pgmap v1160: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 5.4 MiB/s wr, 45 op/s 2024-03-07T03:33:24.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:24 smithi145 bash[79878]: cluster 2024-03-07T03:33:23.292682+0000 mgr.y (mgr.34107) 646 : cluster 0 pgmap v1161: 105 pgs: 105 active+clean; 179 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 5.4 MiB/s wr, 49 op/s 2024-03-07T03:33:24.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:24 smithi145 bash[79878]: cluster 2024-03-07T03:33:23.562999+0000 mon.a (mon.0) 1528 : cluster 0 osdmap e891: 8 total, 8 up, 8 in 2024-03-07T03:33:24.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:24 smithi145 bash[82329]: cluster 2024-03-07T03:33:23.292682+0000 mgr.y (mgr.34107) 646 : cluster 0 pgmap v1161: 105 pgs: 105 active+clean; 179 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 5.4 MiB/s wr, 49 op/s 2024-03-07T03:33:24.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:24 smithi145 bash[82329]: cluster 2024-03-07T03:33:23.562999+0000 mon.a (mon.0) 1528 : cluster 0 osdmap e891: 8 total, 8 up, 8 in 2024-03-07T03:33:24.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:24 smithi189 bash[66969]: cluster 2024-03-07T03:33:23.292682+0000 mgr.y (mgr.34107) 646 : cluster 0 pgmap v1161: 105 pgs: 105 active+clean; 179 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 5.4 MiB/s wr, 49 op/s 2024-03-07T03:33:24.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:24 smithi189 bash[66969]: cluster 2024-03-07T03:33:23.562999+0000 mon.a (mon.0) 1528 : cluster 0 osdmap e891: 8 total, 8 up, 8 in 2024-03-07T03:33:26.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:26 smithi145 bash[79878]: cluster 2024-03-07T03:33:25.293566+0000 mgr.y (mgr.34107) 647 : cluster 0 pgmap v1163: 105 pgs: 105 active+clean; 215 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 8.6 MiB/s wr, 63 op/s 2024-03-07T03:33:26.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:26 smithi145 bash[82329]: cluster 2024-03-07T03:33:25.293566+0000 mgr.y (mgr.34107) 647 : cluster 0 pgmap v1163: 105 pgs: 105 active+clean; 215 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 8.6 MiB/s wr, 63 op/s 2024-03-07T03:33:26.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:26 smithi189 bash[66969]: cluster 2024-03-07T03:33:25.293566+0000 mgr.y (mgr.34107) 647 : cluster 0 pgmap v1163: 105 pgs: 105 active+clean; 215 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 8.6 MiB/s wr, 63 op/s 2024-03-07T03:33:28.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:28 smithi145 bash[79878]: cluster 2024-03-07T03:33:27.294276+0000 mgr.y (mgr.34107) 648 : cluster 0 pgmap v1164: 105 pgs: 105 active+clean; 224 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 63 op/s 2024-03-07T03:33:28.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:28 smithi145 bash[82329]: cluster 2024-03-07T03:33:27.294276+0000 mgr.y (mgr.34107) 648 : cluster 0 pgmap v1164: 105 pgs: 105 active+clean; 224 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 63 op/s 2024-03-07T03:33:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:28 smithi189 bash[66969]: cluster 2024-03-07T03:33:27.294276+0000 mgr.y (mgr.34107) 648 : cluster 0 pgmap v1164: 105 pgs: 105 active+clean; 224 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 63 op/s 2024-03-07T03:33:30.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:30 smithi145 bash[79878]: cluster 2024-03-07T03:33:29.294934+0000 mgr.y (mgr.34107) 649 : cluster 0 pgmap v1165: 105 pgs: 105 active+clean; 224 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.8 MiB/s wr, 68 op/s 2024-03-07T03:33:30.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:30 smithi145 bash[82329]: cluster 2024-03-07T03:33:29.294934+0000 mgr.y (mgr.34107) 649 : cluster 0 pgmap v1165: 105 pgs: 105 active+clean; 224 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.8 MiB/s wr, 68 op/s 2024-03-07T03:33:30.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:30 smithi189 bash[66969]: cluster 2024-03-07T03:33:29.294934+0000 mgr.y (mgr.34107) 649 : cluster 0 pgmap v1165: 105 pgs: 105 active+clean; 224 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.8 MiB/s wr, 68 op/s 2024-03-07T03:33:32.560 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:33:32 smithi189 bash[59814]: ts=2024-03-07T03:33:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:33:32.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:32 smithi189 bash[66969]: cluster 2024-03-07T03:33:31.295585+0000 mgr.y (mgr.34107) 650 : cluster 0 pgmap v1166: 105 pgs: 105 active+clean; 224 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 7.0 MiB/s wr, 64 op/s 2024-03-07T03:33:33.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:32 smithi145 bash[79878]: cluster 2024-03-07T03:33:31.295585+0000 mgr.y (mgr.34107) 650 : cluster 0 pgmap v1166: 105 pgs: 105 active+clean; 224 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 7.0 MiB/s wr, 64 op/s 2024-03-07T03:33:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:32 smithi145 bash[82329]: cluster 2024-03-07T03:33:31.295585+0000 mgr.y (mgr.34107) 650 : cluster 0 pgmap v1166: 105 pgs: 105 active+clean; 224 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 7.0 MiB/s wr, 64 op/s 2024-03-07T03:33:33.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:33:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:33:32] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2024-03-07T03:33:33.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:33 smithi189 bash[66969]: audit 2024-03-07T03:33:32.658624+0000 mon.a (mon.0) 1529 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:34.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:33 smithi145 bash[79878]: audit 2024-03-07T03:33:32.658624+0000 mon.a (mon.0) 1529 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:34.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:33 smithi145 bash[82329]: audit 2024-03-07T03:33:32.658624+0000 mon.a (mon.0) 1529 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:34.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:34 smithi189 bash[66969]: cluster 2024-03-07T03:33:33.296195+0000 mgr.y (mgr.34107) 651 : cluster 0 pgmap v1167: 105 pgs: 105 active+clean; 228 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 6.5 MiB/s wr, 52 op/s 2024-03-07T03:33:34.931 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/1.ShrinkToOne (12141 ms) 2024-03-07T03:33:34.931 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/1.ShrinkToOneAfterSnapshot 2024-03-07T03:33:34.931 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:33:35.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:34 smithi145 bash[79878]: cluster 2024-03-07T03:33:33.296195+0000 mgr.y (mgr.34107) 651 : cluster 0 pgmap v1167: 105 pgs: 105 active+clean; 228 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 6.5 MiB/s wr, 52 op/s 2024-03-07T03:33:35.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:34 smithi145 bash[82329]: cluster 2024-03-07T03:33:33.296195+0000 mgr.y (mgr.34107) 651 : cluster 0 pgmap v1167: 105 pgs: 105 active+clean; 228 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 6.5 MiB/s wr, 52 op/s 2024-03-07T03:33:37.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:36 smithi145 bash[79878]: cluster 2024-03-07T03:33:35.297246+0000 mgr.y (mgr.34107) 652 : cluster 0 pgmap v1168: 105 pgs: 105 active+clean; 236 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 6.9 MiB/s wr, 63 op/s 2024-03-07T03:33:37.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:36 smithi145 bash[79878]: cluster 2024-03-07T03:33:35.603309+0000 mon.a (mon.0) 1530 : cluster 0 osdmap e892: 8 total, 8 up, 8 in 2024-03-07T03:33:37.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:36 smithi145 bash[82329]: cluster 2024-03-07T03:33:35.297246+0000 mgr.y (mgr.34107) 652 : cluster 0 pgmap v1168: 105 pgs: 105 active+clean; 236 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 6.9 MiB/s wr, 63 op/s 2024-03-07T03:33:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:36 smithi145 bash[82329]: cluster 2024-03-07T03:33:35.603309+0000 mon.a (mon.0) 1530 : cluster 0 osdmap e892: 8 total, 8 up, 8 in 2024-03-07T03:33:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:36 smithi189 bash[66969]: cluster 2024-03-07T03:33:35.297246+0000 mgr.y (mgr.34107) 652 : cluster 0 pgmap v1168: 105 pgs: 105 active+clean; 236 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 6.9 MiB/s wr, 63 op/s 2024-03-07T03:33:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:36 smithi189 bash[66969]: cluster 2024-03-07T03:33:35.603309+0000 mon.a (mon.0) 1530 : cluster 0 osdmap e892: 8 total, 8 up, 8 in 2024-03-07T03:33:39.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:38 smithi145 bash[79878]: cluster 2024-03-07T03:33:37.297998+0000 mgr.y (mgr.34107) 653 : cluster 0 pgmap v1170: 105 pgs: 105 active+clean; 252 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.8 MiB/s wr, 62 op/s 2024-03-07T03:33:39.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:38 smithi145 bash[82329]: cluster 2024-03-07T03:33:37.297998+0000 mgr.y (mgr.34107) 653 : cluster 0 pgmap v1170: 105 pgs: 105 active+clean; 252 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.8 MiB/s wr, 62 op/s 2024-03-07T03:33:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:38 smithi189 bash[66969]: cluster 2024-03-07T03:33:37.297998+0000 mgr.y (mgr.34107) 653 : cluster 0 pgmap v1170: 105 pgs: 105 active+clean; 252 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.8 MiB/s wr, 62 op/s 2024-03-07T03:33:41.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:40 smithi145 bash[79878]: cluster 2024-03-07T03:33:39.298685+0000 mgr.y (mgr.34107) 654 : cluster 0 pgmap v1171: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.1 MiB/s wr, 58 op/s 2024-03-07T03:33:41.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:40 smithi145 bash[82329]: cluster 2024-03-07T03:33:39.298685+0000 mgr.y (mgr.34107) 654 : cluster 0 pgmap v1171: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.1 MiB/s wr, 58 op/s 2024-03-07T03:33:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:40 smithi189 bash[66969]: cluster 2024-03-07T03:33:39.298685+0000 mgr.y (mgr.34107) 654 : cluster 0 pgmap v1171: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.1 MiB/s wr, 58 op/s 2024-03-07T03:33:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:33:42 smithi189 bash[59814]: ts=2024-03-07T03:33:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:33:43.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:42 smithi145 bash[79878]: cluster 2024-03-07T03:33:41.299611+0000 mgr.y (mgr.34107) 655 : cluster 0 pgmap v1172: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 60 op/s 2024-03-07T03:33:43.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:42 smithi145 bash[82329]: cluster 2024-03-07T03:33:41.299611+0000 mgr.y (mgr.34107) 655 : cluster 0 pgmap v1172: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 60 op/s 2024-03-07T03:33:43.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:33:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:33:42] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2024-03-07T03:33:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:42 smithi189 bash[66969]: cluster 2024-03-07T03:33:41.299611+0000 mgr.y (mgr.34107) 655 : cluster 0 pgmap v1172: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 60 op/s 2024-03-07T03:33:45.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:44 smithi145 bash[79878]: cluster 2024-03-07T03:33:43.300520+0000 mgr.y (mgr.34107) 656 : cluster 0 pgmap v1173: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.1 MiB/s wr, 59 op/s 2024-03-07T03:33:45.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:44 smithi145 bash[82329]: cluster 2024-03-07T03:33:43.300520+0000 mgr.y (mgr.34107) 656 : cluster 0 pgmap v1173: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.1 MiB/s wr, 59 op/s 2024-03-07T03:33:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:44 smithi189 bash[66969]: cluster 2024-03-07T03:33:43.300520+0000 mgr.y (mgr.34107) 656 : cluster 0 pgmap v1173: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.1 MiB/s wr, 59 op/s 2024-03-07T03:33:47.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:46 smithi145 bash[79878]: cluster 2024-03-07T03:33:45.301836+0000 mgr.y (mgr.34107) 657 : cluster 0 pgmap v1174: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-03-07T03:33:47.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:46 smithi145 bash[82329]: cluster 2024-03-07T03:33:45.301836+0000 mgr.y (mgr.34107) 657 : cluster 0 pgmap v1174: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-03-07T03:33:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:46 smithi189 bash[66969]: cluster 2024-03-07T03:33:45.301836+0000 mgr.y (mgr.34107) 657 : cluster 0 pgmap v1174: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-03-07T03:33:49.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:48 smithi145 bash[79878]: cluster 2024-03-07T03:33:47.302572+0000 mgr.y (mgr.34107) 658 : cluster 0 pgmap v1175: 105 pgs: 105 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.2 MiB/s wr, 38 op/s 2024-03-07T03:33:49.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:48 smithi145 bash[79878]: cluster 2024-03-07T03:33:47.634475+0000 mon.a (mon.0) 1531 : cluster 0 osdmap e893: 8 total, 8 up, 8 in 2024-03-07T03:33:49.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:48 smithi145 bash[79878]: audit 2024-03-07T03:33:47.659347+0000 mon.a (mon.0) 1532 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:48 smithi145 bash[82329]: cluster 2024-03-07T03:33:47.302572+0000 mgr.y (mgr.34107) 658 : cluster 0 pgmap v1175: 105 pgs: 105 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.2 MiB/s wr, 38 op/s 2024-03-07T03:33:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:48 smithi145 bash[82329]: cluster 2024-03-07T03:33:47.634475+0000 mon.a (mon.0) 1531 : cluster 0 osdmap e893: 8 total, 8 up, 8 in 2024-03-07T03:33:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:48 smithi145 bash[82329]: audit 2024-03-07T03:33:47.659347+0000 mon.a (mon.0) 1532 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:48 smithi189 bash[66969]: cluster 2024-03-07T03:33:47.302572+0000 mgr.y (mgr.34107) 658 : cluster 0 pgmap v1175: 105 pgs: 105 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.2 MiB/s wr, 38 op/s 2024-03-07T03:33:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:48 smithi189 bash[66969]: cluster 2024-03-07T03:33:47.634475+0000 mon.a (mon.0) 1531 : cluster 0 osdmap e893: 8 total, 8 up, 8 in 2024-03-07T03:33:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:48 smithi189 bash[66969]: audit 2024-03-07T03:33:47.659347+0000 mon.a (mon.0) 1532 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:50.065 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/1.ShrinkToOneAfterSnapshot (15135 ms) 2024-03-07T03:33:50.065 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/1.MinOverlap 2024-03-07T03:33:50.065 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:33:51.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:50 smithi145 bash[79878]: cluster 2024-03-07T03:33:49.303197+0000 mgr.y (mgr.34107) 659 : cluster 0 pgmap v1177: 105 pgs: 105 active+clean; 301 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.6 MiB/s wr, 24 op/s 2024-03-07T03:33:51.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:50 smithi145 bash[82329]: cluster 2024-03-07T03:33:49.303197+0000 mgr.y (mgr.34107) 659 : cluster 0 pgmap v1177: 105 pgs: 105 active+clean; 301 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.6 MiB/s wr, 24 op/s 2024-03-07T03:33:51.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:50 smithi189 bash[66969]: cluster 2024-03-07T03:33:49.303197+0000 mgr.y (mgr.34107) 659 : cluster 0 pgmap v1177: 105 pgs: 105 active+clean; 301 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.6 MiB/s wr, 24 op/s 2024-03-07T03:33:52.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:51 smithi145 bash[79878]: cluster 2024-03-07T03:33:50.643728+0000 mon.a (mon.0) 1533 : cluster 0 osdmap e894: 8 total, 8 up, 8 in 2024-03-07T03:33:52.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:51 smithi145 bash[82329]: cluster 2024-03-07T03:33:50.643728+0000 mon.a (mon.0) 1533 : cluster 0 osdmap e894: 8 total, 8 up, 8 in 2024-03-07T03:33:52.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:51 smithi189 bash[66969]: cluster 2024-03-07T03:33:50.643728+0000 mon.a (mon.0) 1533 : cluster 0 osdmap e894: 8 total, 8 up, 8 in 2024-03-07T03:33:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:33:52 smithi189 bash[59814]: ts=2024-03-07T03:33:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:33:53.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:52 smithi145 bash[79878]: cluster 2024-03-07T03:33:51.304056+0000 mgr.y (mgr.34107) 660 : cluster 0 pgmap v1179: 105 pgs: 105 active+clean; 329 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.5 MiB/s wr, 37 op/s 2024-03-07T03:33:53.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:33:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:33:52] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.43.0" 2024-03-07T03:33:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:52 smithi145 bash[82329]: cluster 2024-03-07T03:33:51.304056+0000 mgr.y (mgr.34107) 660 : cluster 0 pgmap v1179: 105 pgs: 105 active+clean; 329 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.5 MiB/s wr, 37 op/s 2024-03-07T03:33:53.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:52 smithi189 bash[66969]: cluster 2024-03-07T03:33:51.304056+0000 mgr.y (mgr.34107) 660 : cluster 0 pgmap v1179: 105 pgs: 105 active+clean; 329 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.5 MiB/s wr, 37 op/s 2024-03-07T03:33:54.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:53 smithi145 bash[79878]: audit 2024-03-07T03:33:52.956559+0000 mon.a (mon.0) 1534 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:33:54.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:53 smithi145 bash[82329]: audit 2024-03-07T03:33:52.956559+0000 mon.a (mon.0) 1534 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:33:54.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:53 smithi189 bash[66969]: audit 2024-03-07T03:33:52.956559+0000 mon.a (mon.0) 1534 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:33:55.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:54 smithi145 bash[79878]: cluster 2024-03-07T03:33:53.304812+0000 mgr.y (mgr.34107) 661 : cluster 0 pgmap v1180: 105 pgs: 105 active+clean; 363 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 12 MiB/s wr, 66 op/s 2024-03-07T03:33:55.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:54 smithi145 bash[82329]: cluster 2024-03-07T03:33:53.304812+0000 mgr.y (mgr.34107) 661 : cluster 0 pgmap v1180: 105 pgs: 105 active+clean; 363 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 12 MiB/s wr, 66 op/s 2024-03-07T03:33:55.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:54 smithi189 bash[66969]: cluster 2024-03-07T03:33:53.304812+0000 mgr.y (mgr.34107) 661 : cluster 0 pgmap v1180: 105 pgs: 105 active+clean; 363 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 12 MiB/s wr, 66 op/s 2024-03-07T03:33:57.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:56 smithi145 bash[79878]: cluster 2024-03-07T03:33:55.305516+0000 mgr.y (mgr.34107) 662 : cluster 0 pgmap v1181: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 12 MiB/s wr, 93 op/s 2024-03-07T03:33:57.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:56 smithi145 bash[82329]: cluster 2024-03-07T03:33:55.305516+0000 mgr.y (mgr.34107) 662 : cluster 0 pgmap v1181: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 12 MiB/s wr, 93 op/s 2024-03-07T03:33:57.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:56 smithi189 bash[66969]: cluster 2024-03-07T03:33:55.305516+0000 mgr.y (mgr.34107) 662 : cluster 0 pgmap v1181: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 12 MiB/s wr, 93 op/s 2024-03-07T03:33:59.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:58 smithi145 bash[79878]: cluster 2024-03-07T03:33:57.306194+0000 mgr.y (mgr.34107) 663 : cluster 0 pgmap v1182: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 9.7 MiB/s wr, 77 op/s 2024-03-07T03:33:59.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:58 smithi145 bash[82329]: cluster 2024-03-07T03:33:57.306194+0000 mgr.y (mgr.34107) 663 : cluster 0 pgmap v1182: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 9.7 MiB/s wr, 77 op/s 2024-03-07T03:33:59.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:58 smithi189 bash[66969]: cluster 2024-03-07T03:33:57.306194+0000 mgr.y (mgr.34107) 663 : cluster 0 pgmap v1182: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 9.7 MiB/s wr, 77 op/s 2024-03-07T03:34:00.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[79878]: audit 2024-03-07T03:33:58.897481+0000 mon.a (mon.0) 1535 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[79878]: audit 2024-03-07T03:33:58.905196+0000 mon.a (mon.0) 1536 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[79878]: audit 2024-03-07T03:33:58.999524+0000 mon.a (mon.0) 1537 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[79878]: audit 2024-03-07T03:33:59.007599+0000 mon.a (mon.0) 1538 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[79878]: audit 2024-03-07T03:33:59.636436+0000 mon.a (mon.0) 1539 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:34:00.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[79878]: audit 2024-03-07T03:33:59.637614+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:34:00.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[79878]: audit 2024-03-07T03:33:59.644963+0000 mon.a (mon.0) 1541 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[82329]: audit 2024-03-07T03:33:58.897481+0000 mon.a (mon.0) 1535 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[82329]: audit 2024-03-07T03:33:58.905196+0000 mon.a (mon.0) 1536 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[82329]: audit 2024-03-07T03:33:58.999524+0000 mon.a (mon.0) 1537 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[82329]: audit 2024-03-07T03:33:59.007599+0000 mon.a (mon.0) 1538 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[82329]: audit 2024-03-07T03:33:59.636436+0000 mon.a (mon.0) 1539 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:34:00.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[82329]: audit 2024-03-07T03:33:59.637614+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:34:00.310 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:33:59 smithi145 bash[82329]: audit 2024-03-07T03:33:59.644963+0000 mon.a (mon.0) 1541 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:59 smithi189 bash[66969]: audit 2024-03-07T03:33:58.897481+0000 mon.a (mon.0) 1535 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:59 smithi189 bash[66969]: audit 2024-03-07T03:33:58.905196+0000 mon.a (mon.0) 1536 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:59 smithi189 bash[66969]: audit 2024-03-07T03:33:58.999524+0000 mon.a (mon.0) 1537 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:59 smithi189 bash[66969]: audit 2024-03-07T03:33:59.007599+0000 mon.a (mon.0) 1538 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:00.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:59 smithi189 bash[66969]: audit 2024-03-07T03:33:59.636436+0000 mon.a (mon.0) 1539 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:34:00.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:59 smithi189 bash[66969]: audit 2024-03-07T03:33:59.637614+0000 mon.a (mon.0) 1540 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:34:00.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:33:59 smithi189 bash[66969]: audit 2024-03-07T03:33:59.644963+0000 mon.a (mon.0) 1541 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:01.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:00 smithi145 bash[79878]: cluster 2024-03-07T03:33:59.307092+0000 mgr.y (mgr.34107) 664 : cluster 0 pgmap v1183: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 9.0 MiB/s wr, 59 op/s 2024-03-07T03:34:01.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:00 smithi145 bash[82329]: cluster 2024-03-07T03:33:59.307092+0000 mgr.y (mgr.34107) 664 : cluster 0 pgmap v1183: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 9.0 MiB/s wr, 59 op/s 2024-03-07T03:34:01.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:00 smithi189 bash[66969]: cluster 2024-03-07T03:33:59.307092+0000 mgr.y (mgr.34107) 664 : cluster 0 pgmap v1183: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 9.0 MiB/s wr, 59 op/s 2024-03-07T03:34:02.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:34:02 smithi189 bash[59814]: ts=2024-03-07T03:34:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:34:02.791 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/1.MinOverlap (12725 ms) 2024-03-07T03:34:02.791 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/1.ZeroOverlap 2024-03-07T03:34:02.791 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:34:03.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:34:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:34:02] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2024-03-07T03:34:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:02 smithi145 bash[79878]: cluster 2024-03-07T03:34:01.307899+0000 mgr.y (mgr.34107) 665 : cluster 0 pgmap v1184: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 6.9 MiB/s wr, 61 op/s 2024-03-07T03:34:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:02 smithi145 bash[79878]: audit 2024-03-07T03:34:02.660132+0000 mon.a (mon.0) 1542 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:02 smithi145 bash[82329]: cluster 2024-03-07T03:34:01.307899+0000 mgr.y (mgr.34107) 665 : cluster 0 pgmap v1184: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 6.9 MiB/s wr, 61 op/s 2024-03-07T03:34:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:02 smithi145 bash[82329]: audit 2024-03-07T03:34:02.660132+0000 mon.a (mon.0) 1542 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:03.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:02 smithi189 bash[66969]: cluster 2024-03-07T03:34:01.307899+0000 mgr.y (mgr.34107) 665 : cluster 0 pgmap v1184: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 6.9 MiB/s wr, 61 op/s 2024-03-07T03:34:03.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:02 smithi189 bash[66969]: audit 2024-03-07T03:34:02.660132+0000 mon.a (mon.0) 1542 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:05.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:04 smithi145 bash[79878]: cluster 2024-03-07T03:34:03.308771+0000 mgr.y (mgr.34107) 666 : cluster 0 pgmap v1185: 105 pgs: 105 active+clean; 395 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.8 MiB/s wr, 54 op/s 2024-03-07T03:34:05.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:04 smithi145 bash[79878]: cluster 2024-03-07T03:34:03.931463+0000 mon.a (mon.0) 1543 : cluster 0 osdmap e895: 8 total, 8 up, 8 in 2024-03-07T03:34:05.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:04 smithi145 bash[82329]: cluster 2024-03-07T03:34:03.308771+0000 mgr.y (mgr.34107) 666 : cluster 0 pgmap v1185: 105 pgs: 105 active+clean; 395 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.8 MiB/s wr, 54 op/s 2024-03-07T03:34:05.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:04 smithi145 bash[82329]: cluster 2024-03-07T03:34:03.931463+0000 mon.a (mon.0) 1543 : cluster 0 osdmap e895: 8 total, 8 up, 8 in 2024-03-07T03:34:05.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:04 smithi189 bash[66969]: cluster 2024-03-07T03:34:03.308771+0000 mgr.y (mgr.34107) 666 : cluster 0 pgmap v1185: 105 pgs: 105 active+clean; 395 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.8 MiB/s wr, 54 op/s 2024-03-07T03:34:05.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:04 smithi189 bash[66969]: cluster 2024-03-07T03:34:03.931463+0000 mon.a (mon.0) 1543 : cluster 0 osdmap e895: 8 total, 8 up, 8 in 2024-03-07T03:34:07.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:06 smithi145 bash[79878]: cluster 2024-03-07T03:34:05.309413+0000 mgr.y (mgr.34107) 667 : cluster 0 pgmap v1187: 105 pgs: 105 active+clean; 412 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 5.3 MiB/s wr, 33 op/s 2024-03-07T03:34:07.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:06 smithi145 bash[82329]: cluster 2024-03-07T03:34:05.309413+0000 mgr.y (mgr.34107) 667 : cluster 0 pgmap v1187: 105 pgs: 105 active+clean; 412 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 5.3 MiB/s wr, 33 op/s 2024-03-07T03:34:07.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:06 smithi189 bash[66969]: cluster 2024-03-07T03:34:05.309413+0000 mgr.y (mgr.34107) 667 : cluster 0 pgmap v1187: 105 pgs: 105 active+clean; 412 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 5.3 MiB/s wr, 33 op/s 2024-03-07T03:34:09.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:08 smithi145 bash[79878]: cluster 2024-03-07T03:34:07.310154+0000 mgr.y (mgr.34107) 668 : cluster 0 pgmap v1188: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 42 op/s 2024-03-07T03:34:09.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:08 smithi145 bash[82329]: cluster 2024-03-07T03:34:07.310154+0000 mgr.y (mgr.34107) 668 : cluster 0 pgmap v1188: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 42 op/s 2024-03-07T03:34:09.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:08 smithi189 bash[66969]: cluster 2024-03-07T03:34:07.310154+0000 mgr.y (mgr.34107) 668 : cluster 0 pgmap v1188: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 42 op/s 2024-03-07T03:34:11.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:10 smithi145 bash[79878]: cluster 2024-03-07T03:34:09.310801+0000 mgr.y (mgr.34107) 669 : cluster 0 pgmap v1189: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 5.3 MiB/s wr, 61 op/s 2024-03-07T03:34:11.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:10 smithi145 bash[82329]: cluster 2024-03-07T03:34:09.310801+0000 mgr.y (mgr.34107) 669 : cluster 0 pgmap v1189: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 5.3 MiB/s wr, 61 op/s 2024-03-07T03:34:11.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:10 smithi189 bash[66969]: cluster 2024-03-07T03:34:09.310801+0000 mgr.y (mgr.34107) 669 : cluster 0 pgmap v1189: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 5.3 MiB/s wr, 61 op/s 2024-03-07T03:34:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:34:12 smithi189 bash[59814]: ts=2024-03-07T03:34:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:34:13.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:12 smithi145 bash[79878]: cluster 2024-03-07T03:34:11.311457+0000 mgr.y (mgr.34107) 670 : cluster 0 pgmap v1190: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 5.3 MiB/s wr, 55 op/s 2024-03-07T03:34:13.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:34:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:34:12] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2024-03-07T03:34:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:12 smithi145 bash[82329]: cluster 2024-03-07T03:34:11.311457+0000 mgr.y (mgr.34107) 670 : cluster 0 pgmap v1190: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 5.3 MiB/s wr, 55 op/s 2024-03-07T03:34:13.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:12 smithi189 bash[66969]: cluster 2024-03-07T03:34:11.311457+0000 mgr.y (mgr.34107) 670 : cluster 0 pgmap v1190: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 5.3 MiB/s wr, 55 op/s 2024-03-07T03:34:15.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:14 smithi145 bash[79878]: cluster 2024-03-07T03:34:13.312070+0000 mgr.y (mgr.34107) 671 : cluster 0 pgmap v1191: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 4.5 MiB/s wr, 48 op/s 2024-03-07T03:34:15.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:14 smithi145 bash[82329]: cluster 2024-03-07T03:34:13.312070+0000 mgr.y (mgr.34107) 671 : cluster 0 pgmap v1191: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 4.5 MiB/s wr, 48 op/s 2024-03-07T03:34:15.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:14 smithi189 bash[66969]: cluster 2024-03-07T03:34:13.312070+0000 mgr.y (mgr.34107) 671 : cluster 0 pgmap v1191: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 4.5 MiB/s wr, 48 op/s 2024-03-07T03:34:17.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:16 smithi145 bash[79878]: cluster 2024-03-07T03:34:15.312834+0000 mgr.y (mgr.34107) 672 : cluster 0 pgmap v1192: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 415 KiB/s rd, 1.1 MiB/s wr, 40 op/s 2024-03-07T03:34:17.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:16 smithi145 bash[82329]: cluster 2024-03-07T03:34:15.312834+0000 mgr.y (mgr.34107) 672 : cluster 0 pgmap v1192: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 415 KiB/s rd, 1.1 MiB/s wr, 40 op/s 2024-03-07T03:34:17.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:16 smithi189 bash[66969]: cluster 2024-03-07T03:34:15.312834+0000 mgr.y (mgr.34107) 672 : cluster 0 pgmap v1192: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 415 KiB/s rd, 1.1 MiB/s wr, 40 op/s 2024-03-07T03:34:17.978 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/1.ZeroOverlap (15188 ms) 2024-03-07T03:34:18.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:17 smithi145 bash[79878]: audit 2024-03-07T03:34:17.659816+0000 mon.a (mon.0) 1544 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:18.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:17 smithi145 bash[82329]: audit 2024-03-07T03:34:17.659816+0000 mon.a (mon.0) 1544 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:18.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:17 smithi189 bash[66969]: audit 2024-03-07T03:34:17.659816+0000 mon.a (mon.0) 1544 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:19.010 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/1 (93253 ms total) 2024-03-07T03:34:19.011 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:34:19.011 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/2, where TypeParam = UnformattedLUKS1 2024-03-07T03:34:19.046 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/2.Simple 2024-03-07T03:34:19.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:19 smithi145 bash[79878]: cluster 2024-03-07T03:34:17.313526+0000 mgr.y (mgr.34107) 673 : cluster 0 pgmap v1193: 105 pgs: 105 active+clean; 432 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 729 KiB/s rd, 2.3 MiB/s wr, 33 op/s 2024-03-07T03:34:19.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:18 smithi145 bash[82329]: cluster 2024-03-07T03:34:17.313526+0000 mgr.y (mgr.34107) 673 : cluster 0 pgmap v1193: 105 pgs: 105 active+clean; 432 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 729 KiB/s rd, 2.3 MiB/s wr, 33 op/s 2024-03-07T03:34:19.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:18 smithi189 bash[66969]: cluster 2024-03-07T03:34:17.313526+0000 mgr.y (mgr.34107) 673 : cluster 0 pgmap v1193: 105 pgs: 105 active+clean; 432 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 729 KiB/s rd, 2.3 MiB/s wr, 33 op/s 2024-03-07T03:34:20.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:20 smithi145 bash[79878]: cluster 2024-03-07T03:34:19.014362+0000 mon.a (mon.0) 1545 : cluster 0 osdmap e896: 8 total, 8 up, 8 in 2024-03-07T03:34:20.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:20 smithi145 bash[79878]: cluster 2024-03-07T03:34:19.314170+0000 mgr.y (mgr.34107) 674 : cluster 0 pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:34:20.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:20 smithi145 bash[82329]: cluster 2024-03-07T03:34:19.014362+0000 mon.a (mon.0) 1545 : cluster 0 osdmap e896: 8 total, 8 up, 8 in 2024-03-07T03:34:20.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:20 smithi145 bash[82329]: cluster 2024-03-07T03:34:19.314170+0000 mgr.y (mgr.34107) 674 : cluster 0 pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:34:20.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:20 smithi189 bash[66969]: cluster 2024-03-07T03:34:19.014362+0000 mon.a (mon.0) 1545 : cluster 0 osdmap e896: 8 total, 8 up, 8 in 2024-03-07T03:34:20.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:20 smithi189 bash[66969]: cluster 2024-03-07T03:34:19.314170+0000 mgr.y (mgr.34107) 674 : cluster 0 pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:34:21.031 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:34:21.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:21 smithi145 bash[79878]: cluster 2024-03-07T03:34:20.031328+0000 mon.a (mon.0) 1546 : cluster 0 osdmap e897: 8 total, 8 up, 8 in 2024-03-07T03:34:21.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:21 smithi145 bash[79878]: audit 2024-03-07T03:34:20.036660+0000 mon.c (mon.2) 47 : audit 1 from='client.? 172.21.15.145:0/3625585100' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:34:21.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:21 smithi145 bash[79878]: audit 2024-03-07T03:34:20.037881+0000 mon.a (mon.0) 1547 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:34:21.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:21 smithi145 bash[82329]: cluster 2024-03-07T03:34:20.031328+0000 mon.a (mon.0) 1546 : cluster 0 osdmap e897: 8 total, 8 up, 8 in 2024-03-07T03:34:21.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:21 smithi145 bash[82329]: audit 2024-03-07T03:34:20.036660+0000 mon.c (mon.2) 47 : audit 1 from='client.? 172.21.15.145:0/3625585100' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:34:21.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:21 smithi145 bash[82329]: audit 2024-03-07T03:34:20.037881+0000 mon.a (mon.0) 1547 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:34:21.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:21 smithi189 bash[66969]: cluster 2024-03-07T03:34:20.031328+0000 mon.a (mon.0) 1546 : cluster 0 osdmap e897: 8 total, 8 up, 8 in 2024-03-07T03:34:21.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:21 smithi189 bash[66969]: audit 2024-03-07T03:34:20.036660+0000 mon.c (mon.2) 47 : audit 1 from='client.? 172.21.15.145:0/3625585100' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:34:21.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:21 smithi189 bash[66969]: audit 2024-03-07T03:34:20.037881+0000 mon.a (mon.0) 1547 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:34:22.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:22 smithi189 bash[66969]: audit 2024-03-07T03:34:21.024703+0000 mon.a (mon.0) 1548 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-38","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:34:22.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:22 smithi189 bash[66969]: cluster 2024-03-07T03:34:21.031683+0000 mon.a (mon.0) 1549 : cluster 0 osdmap e898: 8 total, 8 up, 8 in 2024-03-07T03:34:22.303 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:22 smithi189 bash[66969]: cluster 2024-03-07T03:34:21.314847+0000 mgr.y (mgr.34107) 675 : cluster 0 pgmap v1198: 105 pgs: 9 creating+peering, 23 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:34:22.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:22 smithi145 bash[79878]: audit 2024-03-07T03:34:21.024703+0000 mon.a (mon.0) 1548 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-38","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:34:22.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:22 smithi145 bash[79878]: cluster 2024-03-07T03:34:21.031683+0000 mon.a (mon.0) 1549 : cluster 0 osdmap e898: 8 total, 8 up, 8 in 2024-03-07T03:34:22.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:22 smithi145 bash[79878]: cluster 2024-03-07T03:34:21.314847+0000 mgr.y (mgr.34107) 675 : cluster 0 pgmap v1198: 105 pgs: 9 creating+peering, 23 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:34:22.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:22 smithi145 bash[82329]: audit 2024-03-07T03:34:21.024703+0000 mon.a (mon.0) 1548 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-38","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:34:22.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:22 smithi145 bash[82329]: cluster 2024-03-07T03:34:21.031683+0000 mon.a (mon.0) 1549 : cluster 0 osdmap e898: 8 total, 8 up, 8 in 2024-03-07T03:34:22.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:22 smithi145 bash[82329]: cluster 2024-03-07T03:34:21.314847+0000 mgr.y (mgr.34107) 675 : cluster 0 pgmap v1198: 105 pgs: 9 creating+peering, 23 unknown, 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:34:22.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:34:22 smithi189 bash[59814]: ts=2024-03-07T03:34:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:34:23.049 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:34:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:34:22] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.43.0" 2024-03-07T03:34:23.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:23 smithi145 bash[79878]: cluster 2024-03-07T03:34:22.057429+0000 mon.a (mon.0) 1550 : cluster 0 osdmap e899: 8 total, 8 up, 8 in 2024-03-07T03:34:23.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:23 smithi145 bash[82329]: cluster 2024-03-07T03:34:22.057429+0000 mon.a (mon.0) 1550 : cluster 0 osdmap e899: 8 total, 8 up, 8 in 2024-03-07T03:34:23.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:23 smithi189 bash[66969]: cluster 2024-03-07T03:34:22.057429+0000 mon.a (mon.0) 1550 : cluster 0 osdmap e899: 8 total, 8 up, 8 in 2024-03-07T03:34:24.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:24 smithi145 bash[82329]: cluster 2024-03-07T03:34:23.057229+0000 mon.a (mon.0) 1551 : cluster 0 osdmap e900: 8 total, 8 up, 8 in 2024-03-07T03:34:24.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:24 smithi145 bash[82329]: cluster 2024-03-07T03:34:23.315487+0000 mgr.y (mgr.34107) 676 : cluster 0 pgmap v1201: 105 pgs: 9 creating+peering, 17 unknown, 79 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:34:24.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:24 smithi145 bash[79878]: cluster 2024-03-07T03:34:23.057229+0000 mon.a (mon.0) 1551 : cluster 0 osdmap e900: 8 total, 8 up, 8 in 2024-03-07T03:34:24.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:24 smithi145 bash[79878]: cluster 2024-03-07T03:34:23.315487+0000 mgr.y (mgr.34107) 676 : cluster 0 pgmap v1201: 105 pgs: 9 creating+peering, 17 unknown, 79 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:34:24.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:24 smithi189 bash[66969]: cluster 2024-03-07T03:34:23.057229+0000 mon.a (mon.0) 1551 : cluster 0 osdmap e900: 8 total, 8 up, 8 in 2024-03-07T03:34:24.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:24 smithi189 bash[66969]: cluster 2024-03-07T03:34:23.315487+0000 mgr.y (mgr.34107) 676 : cluster 0 pgmap v1201: 105 pgs: 9 creating+peering, 17 unknown, 79 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:34:26.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:26 smithi145 bash[79878]: cluster 2024-03-07T03:34:25.316460+0000 mgr.y (mgr.34107) 677 : cluster 0 pgmap v1202: 105 pgs: 7 creating+peering, 98 active+clean; 582 KiB data, 595 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 967 B/s wr, 8 op/s 2024-03-07T03:34:26.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:26 smithi145 bash[82329]: cluster 2024-03-07T03:34:25.316460+0000 mgr.y (mgr.34107) 677 : cluster 0 pgmap v1202: 105 pgs: 7 creating+peering, 98 active+clean; 582 KiB data, 595 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 967 B/s wr, 8 op/s 2024-03-07T03:34:26.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:26 smithi189 bash[66969]: cluster 2024-03-07T03:34:25.316460+0000 mgr.y (mgr.34107) 677 : cluster 0 pgmap v1202: 105 pgs: 7 creating+peering, 98 active+clean; 582 KiB data, 595 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 967 B/s wr, 8 op/s 2024-03-07T03:34:28.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:28 smithi145 bash[79878]: cluster 2024-03-07T03:34:27.317527+0000 mgr.y (mgr.34107) 678 : cluster 0 pgmap v1203: 105 pgs: 105 active+clean; 582 KiB data, 595 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-03-07T03:34:28.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:28 smithi145 bash[79878]: cluster 2024-03-07T03:34:27.388235+0000 mon.a (mon.0) 1552 : cluster 0 osdmap e901: 8 total, 8 up, 8 in 2024-03-07T03:34:28.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:28 smithi145 bash[79878]: cluster 2024-03-07T03:34:27.603367+0000 mon.a (mon.0) 1553 : cluster 0 osdmap e902: 8 total, 8 up, 8 in 2024-03-07T03:34:28.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:28 smithi145 bash[82329]: cluster 2024-03-07T03:34:27.317527+0000 mgr.y (mgr.34107) 678 : cluster 0 pgmap v1203: 105 pgs: 105 active+clean; 582 KiB data, 595 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-03-07T03:34:28.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:28 smithi145 bash[82329]: cluster 2024-03-07T03:34:27.388235+0000 mon.a (mon.0) 1552 : cluster 0 osdmap e901: 8 total, 8 up, 8 in 2024-03-07T03:34:28.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:28 smithi145 bash[82329]: cluster 2024-03-07T03:34:27.603367+0000 mon.a (mon.0) 1553 : cluster 0 osdmap e902: 8 total, 8 up, 8 in 2024-03-07T03:34:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:28 smithi189 bash[66969]: cluster 2024-03-07T03:34:27.317527+0000 mgr.y (mgr.34107) 678 : cluster 0 pgmap v1203: 105 pgs: 105 active+clean; 582 KiB data, 595 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-03-07T03:34:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:28 smithi189 bash[66969]: cluster 2024-03-07T03:34:27.388235+0000 mon.a (mon.0) 1552 : cluster 0 osdmap e901: 8 total, 8 up, 8 in 2024-03-07T03:34:28.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:28 smithi189 bash[66969]: cluster 2024-03-07T03:34:27.603367+0000 mon.a (mon.0) 1553 : cluster 0 osdmap e902: 8 total, 8 up, 8 in 2024-03-07T03:34:30.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:30 smithi145 bash[79878]: cluster 2024-03-07T03:34:29.318152+0000 mgr.y (mgr.34107) 679 : cluster 0 pgmap v1206: 105 pgs: 105 active+clean; 18 MiB data, 632 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 2.7 MiB/s wr, 53 op/s 2024-03-07T03:34:30.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:30 smithi145 bash[82329]: cluster 2024-03-07T03:34:29.318152+0000 mgr.y (mgr.34107) 679 : cluster 0 pgmap v1206: 105 pgs: 105 active+clean; 18 MiB data, 632 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 2.7 MiB/s wr, 53 op/s 2024-03-07T03:34:30.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:30 smithi189 bash[66969]: cluster 2024-03-07T03:34:29.318152+0000 mgr.y (mgr.34107) 679 : cluster 0 pgmap v1206: 105 pgs: 105 active+clean; 18 MiB data, 632 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 2.7 MiB/s wr, 53 op/s 2024-03-07T03:34:32.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:32 smithi189 bash[66969]: cluster 2024-03-07T03:34:31.318648+0000 mgr.y (mgr.34107) 680 : cluster 0 pgmap v1207: 105 pgs: 105 active+clean; 30 MiB data, 665 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 3.6 MiB/s wr, 53 op/s 2024-03-07T03:34:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:34:32 smithi189 bash[59814]: ts=2024-03-07T03:34:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:34:32.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:32 smithi145 bash[79878]: cluster 2024-03-07T03:34:31.318648+0000 mgr.y (mgr.34107) 680 : cluster 0 pgmap v1207: 105 pgs: 105 active+clean; 30 MiB data, 665 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 3.6 MiB/s wr, 53 op/s 2024-03-07T03:34:32.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:32 smithi145 bash[82329]: cluster 2024-03-07T03:34:31.318648+0000 mgr.y (mgr.34107) 680 : cluster 0 pgmap v1207: 105 pgs: 105 active+clean; 30 MiB data, 665 MiB used, 715 GiB / 715 GiB avail; 112 KiB/s rd, 3.6 MiB/s wr, 53 op/s 2024-03-07T03:34:33.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:34:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:34:32] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.43.0" 2024-03-07T03:34:33.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:33 smithi145 bash[79878]: audit 2024-03-07T03:34:32.660838+0000 mon.a (mon.0) 1554 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:33.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:33 smithi145 bash[82329]: audit 2024-03-07T03:34:32.660838+0000 mon.a (mon.0) 1554 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:33.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:33 smithi189 bash[66969]: audit 2024-03-07T03:34:32.660838+0000 mon.a (mon.0) 1554 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:34.336 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/2.Simple (15290 ms) 2024-03-07T03:34:34.336 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/2.Grow 2024-03-07T03:34:34.336 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:34:34.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:34 smithi145 bash[79878]: cluster 2024-03-07T03:34:33.319471+0000 mgr.y (mgr.34107) 681 : cluster 0 pgmap v1208: 105 pgs: 105 active+clean; 30 MiB data, 682 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 3.6 MiB/s wr, 50 op/s 2024-03-07T03:34:34.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:34 smithi145 bash[82329]: cluster 2024-03-07T03:34:33.319471+0000 mgr.y (mgr.34107) 681 : cluster 0 pgmap v1208: 105 pgs: 105 active+clean; 30 MiB data, 682 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 3.6 MiB/s wr, 50 op/s 2024-03-07T03:34:34.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:34 smithi189 bash[66969]: cluster 2024-03-07T03:34:33.319471+0000 mgr.y (mgr.34107) 681 : cluster 0 pgmap v1208: 105 pgs: 105 active+clean; 30 MiB data, 682 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 3.6 MiB/s wr, 50 op/s 2024-03-07T03:34:35.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:35 smithi145 bash[79878]: audit 2024-03-07T03:34:34.528562+0000 mon.a (mon.0) 1555 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:35.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:35 smithi145 bash[82329]: audit 2024-03-07T03:34:34.528562+0000 mon.a (mon.0) 1555 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:35.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:35 smithi189 bash[66969]: audit 2024-03-07T03:34:34.528562+0000 mon.a (mon.0) 1555 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:34:36.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:36 smithi145 bash[79878]: cluster 2024-03-07T03:34:35.320492+0000 mgr.y (mgr.34107) 682 : cluster 0 pgmap v1209: 105 pgs: 105 active+clean; 55 MiB data, 736 MiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 6.8 MiB/s wr, 63 op/s 2024-03-07T03:34:36.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:36 smithi145 bash[82329]: cluster 2024-03-07T03:34:35.320492+0000 mgr.y (mgr.34107) 682 : cluster 0 pgmap v1209: 105 pgs: 105 active+clean; 55 MiB data, 736 MiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 6.8 MiB/s wr, 63 op/s 2024-03-07T03:34:36.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:36 smithi189 bash[66969]: cluster 2024-03-07T03:34:35.320492+0000 mgr.y (mgr.34107) 682 : cluster 0 pgmap v1209: 105 pgs: 105 active+clean; 55 MiB data, 736 MiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 6.8 MiB/s wr, 63 op/s 2024-03-07T03:34:38.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:38 smithi145 bash[79878]: cluster 2024-03-07T03:34:37.321424+0000 mgr.y (mgr.34107) 683 : cluster 0 pgmap v1210: 105 pgs: 105 active+clean; 59 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 5.8 MiB/s wr, 58 op/s 2024-03-07T03:34:38.812 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:38 smithi145 bash[82329]: cluster 2024-03-07T03:34:37.321424+0000 mgr.y (mgr.34107) 683 : cluster 0 pgmap v1210: 105 pgs: 105 active+clean; 59 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 5.8 MiB/s wr, 58 op/s 2024-03-07T03:34:38.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:38 smithi189 bash[66969]: cluster 2024-03-07T03:34:37.321424+0000 mgr.y (mgr.34107) 683 : cluster 0 pgmap v1210: 105 pgs: 105 active+clean; 59 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 5.8 MiB/s wr, 58 op/s 2024-03-07T03:34:39.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:39 smithi145 bash[82329]: cluster 2024-03-07T03:34:38.557870+0000 mon.a (mon.0) 1556 : cluster 0 osdmap e903: 8 total, 8 up, 8 in 2024-03-07T03:34:39.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:39 smithi145 bash[79878]: cluster 2024-03-07T03:34:38.557870+0000 mon.a (mon.0) 1556 : cluster 0 osdmap e903: 8 total, 8 up, 8 in 2024-03-07T03:34:39.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:39 smithi189 bash[66969]: cluster 2024-03-07T03:34:38.557870+0000 mon.a (mon.0) 1556 : cluster 0 osdmap e903: 8 total, 8 up, 8 in 2024-03-07T03:34:40.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:40 smithi145 bash[79878]: cluster 2024-03-07T03:34:39.322033+0000 mgr.y (mgr.34107) 684 : cluster 0 pgmap v1212: 105 pgs: 105 active+clean; 79 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.1 MiB/s wr, 62 op/s 2024-03-07T03:34:40.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:40 smithi145 bash[82329]: cluster 2024-03-07T03:34:39.322033+0000 mgr.y (mgr.34107) 684 : cluster 0 pgmap v1212: 105 pgs: 105 active+clean; 79 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.1 MiB/s wr, 62 op/s 2024-03-07T03:34:40.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:40 smithi189 bash[66969]: cluster 2024-03-07T03:34:39.322033+0000 mgr.y (mgr.34107) 684 : cluster 0 pgmap v1212: 105 pgs: 105 active+clean; 79 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.1 MiB/s wr, 62 op/s 2024-03-07T03:34:42.553 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:34:42 smithi189 bash[59814]: ts=2024-03-07T03:34:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:34:42.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:42 smithi145 bash[79878]: cluster 2024-03-07T03:34:41.322541+0000 mgr.y (mgr.34107) 685 : cluster 0 pgmap v1213: 105 pgs: 105 active+clean; 83 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.3 MiB/s wr, 54 op/s 2024-03-07T03:34:42.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:34:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:34:42] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.43.0" 2024-03-07T03:34:42.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:42 smithi145 bash[82329]: cluster 2024-03-07T03:34:41.322541+0000 mgr.y (mgr.34107) 685 : cluster 0 pgmap v1213: 105 pgs: 105 active+clean; 83 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.3 MiB/s wr, 54 op/s 2024-03-07T03:34:42.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:42 smithi189 bash[66969]: cluster 2024-03-07T03:34:41.322541+0000 mgr.y (mgr.34107) 685 : cluster 0 pgmap v1213: 105 pgs: 105 active+clean; 83 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.3 MiB/s wr, 54 op/s 2024-03-07T03:34:44.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:44 smithi145 bash[79878]: cluster 2024-03-07T03:34:43.323331+0000 mgr.y (mgr.34107) 686 : cluster 0 pgmap v1214: 105 pgs: 105 active+clean; 88 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 53 op/s 2024-03-07T03:34:44.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:44 smithi145 bash[82329]: cluster 2024-03-07T03:34:43.323331+0000 mgr.y (mgr.34107) 686 : cluster 0 pgmap v1214: 105 pgs: 105 active+clean; 88 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 53 op/s 2024-03-07T03:34:44.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:44 smithi189 bash[66969]: cluster 2024-03-07T03:34:43.323331+0000 mgr.y (mgr.34107) 686 : cluster 0 pgmap v1214: 105 pgs: 105 active+clean; 88 MiB data, 844 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 53 op/s 2024-03-07T03:34:45.532 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/2.Grow (11196 ms) 2024-03-07T03:34:45.532 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/2.Shrink 2024-03-07T03:34:45.532 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:34:46.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:46 smithi189 bash[66969]: cluster 2024-03-07T03:34:45.324428+0000 mgr.y (mgr.34107) 687 : cluster 0 pgmap v1215: 105 pgs: 105 active+clean; 104 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.9 MiB/s wr, 61 op/s 2024-03-07T03:34:47.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:46 smithi145 bash[82329]: cluster 2024-03-07T03:34:45.324428+0000 mgr.y (mgr.34107) 687 : cluster 0 pgmap v1215: 105 pgs: 105 active+clean; 104 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.9 MiB/s wr, 61 op/s 2024-03-07T03:34:47.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:46 smithi145 bash[79878]: cluster 2024-03-07T03:34:45.324428+0000 mgr.y (mgr.34107) 687 : cluster 0 pgmap v1215: 105 pgs: 105 active+clean; 104 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.9 MiB/s wr, 61 op/s 2024-03-07T03:34:48.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:48 smithi189 bash[66969]: cluster 2024-03-07T03:34:47.325400+0000 mgr.y (mgr.34107) 688 : cluster 0 pgmap v1216: 105 pgs: 105 active+clean; 117 MiB data, 935 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 64 op/s 2024-03-07T03:34:48.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:48 smithi189 bash[66969]: audit 2024-03-07T03:34:47.660421+0000 mon.a (mon.0) 1557 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:49.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:48 smithi145 bash[79878]: cluster 2024-03-07T03:34:47.325400+0000 mgr.y (mgr.34107) 688 : cluster 0 pgmap v1216: 105 pgs: 105 active+clean; 117 MiB data, 935 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 64 op/s 2024-03-07T03:34:49.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:48 smithi145 bash[79878]: audit 2024-03-07T03:34:47.660421+0000 mon.a (mon.0) 1557 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:48 smithi145 bash[82329]: cluster 2024-03-07T03:34:47.325400+0000 mgr.y (mgr.34107) 688 : cluster 0 pgmap v1216: 105 pgs: 105 active+clean; 117 MiB data, 935 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 64 op/s 2024-03-07T03:34:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:48 smithi145 bash[82329]: audit 2024-03-07T03:34:47.660421+0000 mon.a (mon.0) 1557 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:50.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:50 smithi189 bash[66969]: cluster 2024-03-07T03:34:49.326430+0000 mgr.y (mgr.34107) 689 : cluster 0 pgmap v1217: 105 pgs: 105 active+clean; 117 MiB data, 943 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.4 MiB/s wr, 64 op/s 2024-03-07T03:34:50.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:50 smithi189 bash[66969]: cluster 2024-03-07T03:34:49.591679+0000 mon.a (mon.0) 1558 : cluster 0 osdmap e904: 8 total, 8 up, 8 in 2024-03-07T03:34:51.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:50 smithi145 bash[79878]: cluster 2024-03-07T03:34:49.326430+0000 mgr.y (mgr.34107) 689 : cluster 0 pgmap v1217: 105 pgs: 105 active+clean; 117 MiB data, 943 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.4 MiB/s wr, 64 op/s 2024-03-07T03:34:51.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:50 smithi145 bash[79878]: cluster 2024-03-07T03:34:49.591679+0000 mon.a (mon.0) 1558 : cluster 0 osdmap e904: 8 total, 8 up, 8 in 2024-03-07T03:34:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:50 smithi145 bash[82329]: cluster 2024-03-07T03:34:49.326430+0000 mgr.y (mgr.34107) 689 : cluster 0 pgmap v1217: 105 pgs: 105 active+clean; 117 MiB data, 943 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.4 MiB/s wr, 64 op/s 2024-03-07T03:34:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:50 smithi145 bash[82329]: cluster 2024-03-07T03:34:49.591679+0000 mon.a (mon.0) 1558 : cluster 0 osdmap e904: 8 total, 8 up, 8 in 2024-03-07T03:34:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:34:52 smithi189 bash[59814]: ts=2024-03-07T03:34:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:34:53.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:34:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:34:52] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2024-03-07T03:34:53.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:52 smithi145 bash[82329]: cluster 2024-03-07T03:34:51.327367+0000 mgr.y (mgr.34107) 690 : cluster 0 pgmap v1219: 105 pgs: 105 active+clean; 129 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4.6 MiB/s wr, 38 op/s 2024-03-07T03:34:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:52 smithi145 bash[79878]: cluster 2024-03-07T03:34:51.327367+0000 mgr.y (mgr.34107) 690 : cluster 0 pgmap v1219: 105 pgs: 105 active+clean; 129 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4.6 MiB/s wr, 38 op/s 2024-03-07T03:34:53.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:52 smithi189 bash[66969]: cluster 2024-03-07T03:34:51.327367+0000 mgr.y (mgr.34107) 690 : cluster 0 pgmap v1219: 105 pgs: 105 active+clean; 129 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4.6 MiB/s wr, 38 op/s 2024-03-07T03:34:55.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:54 smithi145 bash[79878]: cluster 2024-03-07T03:34:53.328160+0000 mgr.y (mgr.34107) 691 : cluster 0 pgmap v1220: 105 pgs: 105 active+clean; 129 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:34:55.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:54 smithi145 bash[82329]: cluster 2024-03-07T03:34:53.328160+0000 mgr.y (mgr.34107) 691 : cluster 0 pgmap v1220: 105 pgs: 105 active+clean; 129 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:34:55.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:54 smithi189 bash[66969]: cluster 2024-03-07T03:34:53.328160+0000 mgr.y (mgr.34107) 691 : cluster 0 pgmap v1220: 105 pgs: 105 active+clean; 129 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:34:56.540 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/2.Shrink (11009 ms) 2024-03-07T03:34:56.541 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/2.ShrinkToOne 2024-03-07T03:34:56.541 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:34:57.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:56 smithi145 bash[82329]: cluster 2024-03-07T03:34:55.329203+0000 mgr.y (mgr.34107) 692 : cluster 0 pgmap v1221: 105 pgs: 105 active+clean; 146 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 4.2 MiB/s wr, 55 op/s 2024-03-07T03:34:57.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:56 smithi145 bash[79878]: cluster 2024-03-07T03:34:55.329203+0000 mgr.y (mgr.34107) 692 : cluster 0 pgmap v1221: 105 pgs: 105 active+clean; 146 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 4.2 MiB/s wr, 55 op/s 2024-03-07T03:34:57.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:56 smithi189 bash[66969]: cluster 2024-03-07T03:34:55.329203+0000 mgr.y (mgr.34107) 692 : cluster 0 pgmap v1221: 105 pgs: 105 active+clean; 146 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 4.2 MiB/s wr, 55 op/s 2024-03-07T03:34:59.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:34:58 smithi145 bash[79878]: cluster 2024-03-07T03:34:57.330219+0000 mgr.y (mgr.34107) 693 : cluster 0 pgmap v1222: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.3 MiB/s wr, 55 op/s 2024-03-07T03:34:59.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:34:58 smithi145 bash[82329]: cluster 2024-03-07T03:34:57.330219+0000 mgr.y (mgr.34107) 693 : cluster 0 pgmap v1222: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.3 MiB/s wr, 55 op/s 2024-03-07T03:34:59.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:34:58 smithi189 bash[66969]: cluster 2024-03-07T03:34:57.330219+0000 mgr.y (mgr.34107) 693 : cluster 0 pgmap v1222: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.3 MiB/s wr, 55 op/s 2024-03-07T03:35:01.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:00 smithi145 bash[79878]: cluster 2024-03-07T03:34:59.331098+0000 mgr.y (mgr.34107) 694 : cluster 0 pgmap v1223: 105 pgs: 105 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 5.8 MiB/s wr, 61 op/s 2024-03-07T03:35:01.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:00 smithi145 bash[79878]: audit 2024-03-07T03:34:59.725360+0000 mon.a (mon.0) 1559 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:35:01.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:00 smithi145 bash[82329]: cluster 2024-03-07T03:34:59.331098+0000 mgr.y (mgr.34107) 694 : cluster 0 pgmap v1223: 105 pgs: 105 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 5.8 MiB/s wr, 61 op/s 2024-03-07T03:35:01.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:00 smithi145 bash[82329]: audit 2024-03-07T03:34:59.725360+0000 mon.a (mon.0) 1559 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:35:01.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:00 smithi189 bash[66969]: cluster 2024-03-07T03:34:59.331098+0000 mgr.y (mgr.34107) 694 : cluster 0 pgmap v1223: 105 pgs: 105 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 5.8 MiB/s wr, 61 op/s 2024-03-07T03:35:01.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:00 smithi189 bash[66969]: audit 2024-03-07T03:34:59.725360+0000 mon.a (mon.0) 1559 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:35:02.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:01 smithi145 bash[79878]: cluster 2024-03-07T03:35:00.619784+0000 mon.a (mon.0) 1560 : cluster 0 osdmap e905: 8 total, 8 up, 8 in 2024-03-07T03:35:02.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:01 smithi145 bash[82329]: cluster 2024-03-07T03:35:00.619784+0000 mon.a (mon.0) 1560 : cluster 0 osdmap e905: 8 total, 8 up, 8 in 2024-03-07T03:35:02.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:01 smithi189 bash[66969]: cluster 2024-03-07T03:35:00.619784+0000 mon.a (mon.0) 1560 : cluster 0 osdmap e905: 8 total, 8 up, 8 in 2024-03-07T03:35:02.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:35:02 smithi189 bash[59814]: ts=2024-03-07T03:35:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:35:03.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:02 smithi145 bash[79878]: cluster 2024-03-07T03:35:01.331652+0000 mgr.y (mgr.34107) 695 : cluster 0 pgmap v1225: 105 pgs: 105 active+clean; 179 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 60 op/s 2024-03-07T03:35:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:02 smithi145 bash[82329]: cluster 2024-03-07T03:35:01.331652+0000 mgr.y (mgr.34107) 695 : cluster 0 pgmap v1225: 105 pgs: 105 active+clean; 179 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 60 op/s 2024-03-07T03:35:03.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:35:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:35:02] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2024-03-07T03:35:03.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:02 smithi189 bash[66969]: cluster 2024-03-07T03:35:01.331652+0000 mgr.y (mgr.34107) 695 : cluster 0 pgmap v1225: 105 pgs: 105 active+clean; 179 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 60 op/s 2024-03-07T03:35:04.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:03 smithi145 bash[79878]: audit 2024-03-07T03:35:02.661404+0000 mon.a (mon.0) 1561 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:04.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:03 smithi145 bash[82329]: audit 2024-03-07T03:35:02.661404+0000 mon.a (mon.0) 1561 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:04.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:03 smithi189 bash[66969]: audit 2024-03-07T03:35:02.661404+0000 mon.a (mon.0) 1561 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:05.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:04 smithi145 bash[79878]: cluster 2024-03-07T03:35:03.332499+0000 mgr.y (mgr.34107) 696 : cluster 0 pgmap v1226: 105 pgs: 105 active+clean; 179 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.0 MiB/s wr, 83 op/s 2024-03-07T03:35:05.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:04 smithi145 bash[82329]: cluster 2024-03-07T03:35:03.332499+0000 mgr.y (mgr.34107) 696 : cluster 0 pgmap v1226: 105 pgs: 105 active+clean; 179 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.0 MiB/s wr, 83 op/s 2024-03-07T03:35:05.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:04 smithi189 bash[66969]: cluster 2024-03-07T03:35:03.332499+0000 mgr.y (mgr.34107) 696 : cluster 0 pgmap v1226: 105 pgs: 105 active+clean; 179 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.0 MiB/s wr, 83 op/s 2024-03-07T03:35:06.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:06 smithi145 bash[82329]: cluster 2024-03-07T03:35:05.333168+0000 mgr.y (mgr.34107) 697 : cluster 0 pgmap v1227: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 65 op/s 2024-03-07T03:35:06.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:06 smithi145 bash[82329]: audit 2024-03-07T03:35:05.472829+0000 mon.a (mon.0) 1562 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:06.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:06 smithi145 bash[82329]: audit 2024-03-07T03:35:05.479996+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:06.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:06 smithi145 bash[82329]: audit 2024-03-07T03:35:05.838678+0000 mon.a (mon.0) 1564 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:06.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:06 smithi145 bash[82329]: audit 2024-03-07T03:35:05.845578+0000 mon.a (mon.0) 1565 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:06 smithi145 bash[79878]: cluster 2024-03-07T03:35:05.333168+0000 mgr.y (mgr.34107) 697 : cluster 0 pgmap v1227: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 65 op/s 2024-03-07T03:35:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:06 smithi145 bash[79878]: audit 2024-03-07T03:35:05.472829+0000 mon.a (mon.0) 1562 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:06 smithi145 bash[79878]: audit 2024-03-07T03:35:05.479996+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:06 smithi145 bash[79878]: audit 2024-03-07T03:35:05.838678+0000 mon.a (mon.0) 1564 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:06.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:06 smithi145 bash[79878]: audit 2024-03-07T03:35:05.845578+0000 mon.a (mon.0) 1565 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:06.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:06 smithi189 bash[66969]: cluster 2024-03-07T03:35:05.333168+0000 mgr.y (mgr.34107) 697 : cluster 0 pgmap v1227: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 65 op/s 2024-03-07T03:35:06.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:06 smithi189 bash[66969]: audit 2024-03-07T03:35:05.472829+0000 mon.a (mon.0) 1562 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:06.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:06 smithi189 bash[66969]: audit 2024-03-07T03:35:05.479996+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:06.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:06 smithi189 bash[66969]: audit 2024-03-07T03:35:05.838678+0000 mon.a (mon.0) 1564 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:06.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:06 smithi189 bash[66969]: audit 2024-03-07T03:35:05.845578+0000 mon.a (mon.0) 1565 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:07.404 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/2.ShrinkToOne (10863 ms) 2024-03-07T03:35:07.404 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/2.ShrinkToOneAfterSnapshot 2024-03-07T03:35:07.404 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:35:07.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:07 smithi145 bash[79878]: audit 2024-03-07T03:35:06.480611+0000 mon.a (mon.0) 1566 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:35:07.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:07 smithi145 bash[79878]: audit 2024-03-07T03:35:06.482360+0000 mon.a (mon.0) 1567 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:35:07.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:07 smithi145 bash[79878]: audit 2024-03-07T03:35:06.491452+0000 mon.a (mon.0) 1568 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:07.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:07 smithi145 bash[82329]: audit 2024-03-07T03:35:06.480611+0000 mon.a (mon.0) 1566 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:35:07.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:07 smithi145 bash[82329]: audit 2024-03-07T03:35:06.482360+0000 mon.a (mon.0) 1567 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:35:07.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:07 smithi145 bash[82329]: audit 2024-03-07T03:35:06.491452+0000 mon.a (mon.0) 1568 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:07.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:07 smithi189 bash[66969]: audit 2024-03-07T03:35:06.480611+0000 mon.a (mon.0) 1566 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:35:07.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:07 smithi189 bash[66969]: audit 2024-03-07T03:35:06.482360+0000 mon.a (mon.0) 1567 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:35:07.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:07 smithi189 bash[66969]: audit 2024-03-07T03:35:06.491452+0000 mon.a (mon.0) 1568 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:08.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:08 smithi145 bash[79878]: cluster 2024-03-07T03:35:07.333686+0000 mgr.y (mgr.34107) 698 : cluster 0 pgmap v1228: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 3.4 MiB/s wr, 57 op/s 2024-03-07T03:35:08.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:08 smithi145 bash[82329]: cluster 2024-03-07T03:35:07.333686+0000 mgr.y (mgr.34107) 698 : cluster 0 pgmap v1228: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 3.4 MiB/s wr, 57 op/s 2024-03-07T03:35:08.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:08 smithi189 bash[66969]: cluster 2024-03-07T03:35:07.333686+0000 mgr.y (mgr.34107) 698 : cluster 0 pgmap v1228: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 3.4 MiB/s wr, 57 op/s 2024-03-07T03:35:10.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:10 smithi145 bash[79878]: cluster 2024-03-07T03:35:09.334681+0000 mgr.y (mgr.34107) 699 : cluster 0 pgmap v1229: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 58 op/s 2024-03-07T03:35:10.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:10 smithi145 bash[82329]: cluster 2024-03-07T03:35:09.334681+0000 mgr.y (mgr.34107) 699 : cluster 0 pgmap v1229: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 58 op/s 2024-03-07T03:35:10.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:10 smithi189 bash[66969]: cluster 2024-03-07T03:35:09.334681+0000 mgr.y (mgr.34107) 699 : cluster 0 pgmap v1229: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 58 op/s 2024-03-07T03:35:12.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:12 smithi189 bash[66969]: cluster 2024-03-07T03:35:11.335603+0000 mgr.y (mgr.34107) 700 : cluster 0 pgmap v1230: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.8 MiB/s wr, 56 op/s 2024-03-07T03:35:12.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:12 smithi189 bash[66969]: cluster 2024-03-07T03:35:11.510928+0000 mon.a (mon.0) 1569 : cluster 0 osdmap e906: 8 total, 8 up, 8 in 2024-03-07T03:35:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:35:12 smithi189 bash[59814]: ts=2024-03-07T03:35:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:35:12.749 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:12 smithi145 bash[82329]: cluster 2024-03-07T03:35:11.335603+0000 mgr.y (mgr.34107) 700 : cluster 0 pgmap v1230: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.8 MiB/s wr, 56 op/s 2024-03-07T03:35:12.750 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:12 smithi145 bash[82329]: cluster 2024-03-07T03:35:11.510928+0000 mon.a (mon.0) 1569 : cluster 0 osdmap e906: 8 total, 8 up, 8 in 2024-03-07T03:35:12.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:12 smithi145 bash[79878]: cluster 2024-03-07T03:35:11.335603+0000 mgr.y (mgr.34107) 700 : cluster 0 pgmap v1230: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.8 MiB/s wr, 56 op/s 2024-03-07T03:35:12.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:12 smithi145 bash[79878]: cluster 2024-03-07T03:35:11.510928+0000 mon.a (mon.0) 1569 : cluster 0 osdmap e906: 8 total, 8 up, 8 in 2024-03-07T03:35:13.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:35:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:35:12] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2024-03-07T03:35:14.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:14 smithi145 bash[79878]: cluster 2024-03-07T03:35:13.336328+0000 mgr.y (mgr.34107) 701 : cluster 0 pgmap v1232: 105 pgs: 105 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 928 KiB/s rd, 4.1 MiB/s wr, 34 op/s 2024-03-07T03:35:14.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:14 smithi145 bash[82329]: cluster 2024-03-07T03:35:13.336328+0000 mgr.y (mgr.34107) 701 : cluster 0 pgmap v1232: 105 pgs: 105 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 928 KiB/s rd, 4.1 MiB/s wr, 34 op/s 2024-03-07T03:35:14.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:14 smithi189 bash[66969]: cluster 2024-03-07T03:35:13.336328+0000 mgr.y (mgr.34107) 701 : cluster 0 pgmap v1232: 105 pgs: 105 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 928 KiB/s rd, 4.1 MiB/s wr, 34 op/s 2024-03-07T03:35:16.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:16 smithi145 bash[79878]: cluster 2024-03-07T03:35:15.337331+0000 mgr.y (mgr.34107) 702 : cluster 0 pgmap v1233: 105 pgs: 105 active+clean; 233 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 876 KiB/s rd, 3.3 MiB/s wr, 39 op/s 2024-03-07T03:35:16.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:16 smithi145 bash[82329]: cluster 2024-03-07T03:35:15.337331+0000 mgr.y (mgr.34107) 702 : cluster 0 pgmap v1233: 105 pgs: 105 active+clean; 233 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 876 KiB/s rd, 3.3 MiB/s wr, 39 op/s 2024-03-07T03:35:16.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:16 smithi189 bash[66969]: cluster 2024-03-07T03:35:15.337331+0000 mgr.y (mgr.34107) 702 : cluster 0 pgmap v1233: 105 pgs: 105 active+clean; 233 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 876 KiB/s rd, 3.3 MiB/s wr, 39 op/s 2024-03-07T03:35:17.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:17 smithi145 bash[82329]: cluster 2024-03-07T03:35:16.522704+0000 mon.a (mon.0) 1570 : cluster 0 osdmap e907: 8 total, 8 up, 8 in 2024-03-07T03:35:17.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:17 smithi145 bash[79878]: cluster 2024-03-07T03:35:16.522704+0000 mon.a (mon.0) 1570 : cluster 0 osdmap e907: 8 total, 8 up, 8 in 2024-03-07T03:35:17.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:17 smithi189 bash[66969]: cluster 2024-03-07T03:35:16.522704+0000 mon.a (mon.0) 1570 : cluster 0 osdmap e907: 8 total, 8 up, 8 in 2024-03-07T03:35:18.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:18 smithi145 bash[82329]: cluster 2024-03-07T03:35:17.338078+0000 mgr.y (mgr.34107) 703 : cluster 0 pgmap v1235: 105 pgs: 105 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 3.6 MiB/s wr, 51 op/s 2024-03-07T03:35:18.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:18 smithi145 bash[82329]: audit 2024-03-07T03:35:17.661813+0000 mon.a (mon.0) 1571 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:18.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:18 smithi145 bash[79878]: cluster 2024-03-07T03:35:17.338078+0000 mgr.y (mgr.34107) 703 : cluster 0 pgmap v1235: 105 pgs: 105 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 3.6 MiB/s wr, 51 op/s 2024-03-07T03:35:18.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:18 smithi145 bash[79878]: audit 2024-03-07T03:35:17.661813+0000 mon.a (mon.0) 1571 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:18.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:18 smithi189 bash[66969]: cluster 2024-03-07T03:35:17.338078+0000 mgr.y (mgr.34107) 703 : cluster 0 pgmap v1235: 105 pgs: 105 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 3.6 MiB/s wr, 51 op/s 2024-03-07T03:35:18.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:18 smithi189 bash[66969]: audit 2024-03-07T03:35:17.661813+0000 mon.a (mon.0) 1571 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:19.076 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/2.ShrinkToOneAfterSnapshot (11673 ms) 2024-03-07T03:35:19.076 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/2.MinOverlap 2024-03-07T03:35:19.076 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:35:20.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:20 smithi145 bash[79878]: cluster 2024-03-07T03:35:19.339026+0000 mgr.y (mgr.34107) 704 : cluster 0 pgmap v1236: 105 pgs: 105 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 6.3 MiB/s wr, 68 op/s 2024-03-07T03:35:20.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:20 smithi145 bash[82329]: cluster 2024-03-07T03:35:19.339026+0000 mgr.y (mgr.34107) 704 : cluster 0 pgmap v1236: 105 pgs: 105 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 6.3 MiB/s wr, 68 op/s 2024-03-07T03:35:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:20 smithi189 bash[66969]: cluster 2024-03-07T03:35:19.339026+0000 mgr.y (mgr.34107) 704 : cluster 0 pgmap v1236: 105 pgs: 105 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 6.3 MiB/s wr, 68 op/s 2024-03-07T03:35:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:22 smithi189 bash[66969]: cluster 2024-03-07T03:35:21.339865+0000 mgr.y (mgr.34107) 705 : cluster 0 pgmap v1237: 105 pgs: 105 active+clean; 294 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 7.9 MiB/s wr, 71 op/s 2024-03-07T03:35:22.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:35:22 smithi189 bash[59814]: ts=2024-03-07T03:35:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:35:22.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:22 smithi145 bash[79878]: cluster 2024-03-07T03:35:21.339865+0000 mgr.y (mgr.34107) 705 : cluster 0 pgmap v1237: 105 pgs: 105 active+clean; 294 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 7.9 MiB/s wr, 71 op/s 2024-03-07T03:35:22.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:35:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:35:22] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.43.0" 2024-03-07T03:35:22.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:22 smithi145 bash[82329]: cluster 2024-03-07T03:35:21.339865+0000 mgr.y (mgr.34107) 705 : cluster 0 pgmap v1237: 105 pgs: 105 active+clean; 294 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 7.9 MiB/s wr, 71 op/s 2024-03-07T03:35:24.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:24 smithi145 bash[82329]: cluster 2024-03-07T03:35:23.340823+0000 mgr.y (mgr.34107) 706 : cluster 0 pgmap v1238: 105 pgs: 105 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 8.6 MiB/s wr, 74 op/s 2024-03-07T03:35:24.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:24 smithi145 bash[82329]: cluster 2024-03-07T03:35:23.565560+0000 mon.a (mon.0) 1572 : cluster 0 osdmap e908: 8 total, 8 up, 8 in 2024-03-07T03:35:24.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:24 smithi145 bash[79878]: cluster 2024-03-07T03:35:23.340823+0000 mgr.y (mgr.34107) 706 : cluster 0 pgmap v1238: 105 pgs: 105 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 8.6 MiB/s wr, 74 op/s 2024-03-07T03:35:24.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:24 smithi145 bash[79878]: cluster 2024-03-07T03:35:23.565560+0000 mon.a (mon.0) 1572 : cluster 0 osdmap e908: 8 total, 8 up, 8 in 2024-03-07T03:35:24.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:24 smithi189 bash[66969]: cluster 2024-03-07T03:35:23.340823+0000 mgr.y (mgr.34107) 706 : cluster 0 pgmap v1238: 105 pgs: 105 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 8.6 MiB/s wr, 74 op/s 2024-03-07T03:35:24.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:24 smithi189 bash[66969]: cluster 2024-03-07T03:35:23.565560+0000 mon.a (mon.0) 1572 : cluster 0 osdmap e908: 8 total, 8 up, 8 in 2024-03-07T03:35:26.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:26 smithi145 bash[79878]: cluster 2024-03-07T03:35:25.341881+0000 mgr.y (mgr.34107) 707 : cluster 0 pgmap v1240: 105 pgs: 105 active+clean; 322 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.2 MiB/s wr, 49 op/s 2024-03-07T03:35:26.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:26 smithi145 bash[82329]: cluster 2024-03-07T03:35:25.341881+0000 mgr.y (mgr.34107) 707 : cluster 0 pgmap v1240: 105 pgs: 105 active+clean; 322 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.2 MiB/s wr, 49 op/s 2024-03-07T03:35:26.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:26 smithi189 bash[66969]: cluster 2024-03-07T03:35:25.341881+0000 mgr.y (mgr.34107) 707 : cluster 0 pgmap v1240: 105 pgs: 105 active+clean; 322 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.2 MiB/s wr, 49 op/s 2024-03-07T03:35:28.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:28 smithi145 bash[79878]: cluster 2024-03-07T03:35:27.342852+0000 mgr.y (mgr.34107) 708 : cluster 0 pgmap v1241: 105 pgs: 105 active+clean; 331 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 9.0 MiB/s wr, 75 op/s 2024-03-07T03:35:28.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:28 smithi145 bash[82329]: cluster 2024-03-07T03:35:27.342852+0000 mgr.y (mgr.34107) 708 : cluster 0 pgmap v1241: 105 pgs: 105 active+clean; 331 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 9.0 MiB/s wr, 75 op/s 2024-03-07T03:35:28.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:28 smithi189 bash[66969]: cluster 2024-03-07T03:35:27.342852+0000 mgr.y (mgr.34107) 708 : cluster 0 pgmap v1241: 105 pgs: 105 active+clean; 331 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 9.0 MiB/s wr, 75 op/s 2024-03-07T03:35:30.454 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/2.MinOverlap (11377 ms) 2024-03-07T03:35:30.454 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/2.ZeroOverlap 2024-03-07T03:35:30.454 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:35:30.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:30 smithi189 bash[66969]: cluster 2024-03-07T03:35:29.343405+0000 mgr.y (mgr.34107) 709 : cluster 0 pgmap v1242: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 8.1 MiB/s wr, 61 op/s 2024-03-07T03:35:31.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:30 smithi145 bash[79878]: cluster 2024-03-07T03:35:29.343405+0000 mgr.y (mgr.34107) 709 : cluster 0 pgmap v1242: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 8.1 MiB/s wr, 61 op/s 2024-03-07T03:35:31.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:30 smithi145 bash[82329]: cluster 2024-03-07T03:35:29.343405+0000 mgr.y (mgr.34107) 709 : cluster 0 pgmap v1242: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 8.1 MiB/s wr, 61 op/s 2024-03-07T03:35:32.564 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:35:32 smithi189 bash[59814]: ts=2024-03-07T03:35:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:35:32.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:32 smithi189 bash[66969]: cluster 2024-03-07T03:35:31.344263+0000 mgr.y (mgr.34107) 710 : cluster 0 pgmap v1243: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.9 MiB/s wr, 50 op/s 2024-03-07T03:35:33.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:32 smithi145 bash[82329]: cluster 2024-03-07T03:35:31.344263+0000 mgr.y (mgr.34107) 710 : cluster 0 pgmap v1243: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.9 MiB/s wr, 50 op/s 2024-03-07T03:35:33.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:32 smithi145 bash[79878]: cluster 2024-03-07T03:35:31.344263+0000 mgr.y (mgr.34107) 710 : cluster 0 pgmap v1243: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.9 MiB/s wr, 50 op/s 2024-03-07T03:35:33.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:35:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:35:32] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.43.0" 2024-03-07T03:35:33.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:33 smithi189 bash[66969]: audit 2024-03-07T03:35:32.662139+0000 mon.a (mon.0) 1573 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:34.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:33 smithi145 bash[79878]: audit 2024-03-07T03:35:32.662139+0000 mon.a (mon.0) 1573 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:34.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:33 smithi145 bash[82329]: audit 2024-03-07T03:35:32.662139+0000 mon.a (mon.0) 1573 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:35.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:34 smithi145 bash[79878]: cluster 2024-03-07T03:35:33.345145+0000 mgr.y (mgr.34107) 711 : cluster 0 pgmap v1244: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 4.1 MiB/s wr, 59 op/s 2024-03-07T03:35:35.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:34 smithi145 bash[82329]: cluster 2024-03-07T03:35:33.345145+0000 mgr.y (mgr.34107) 711 : cluster 0 pgmap v1244: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 4.1 MiB/s wr, 59 op/s 2024-03-07T03:35:35.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:34 smithi189 bash[66969]: cluster 2024-03-07T03:35:33.345145+0000 mgr.y (mgr.34107) 711 : cluster 0 pgmap v1244: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 4.1 MiB/s wr, 59 op/s 2024-03-07T03:35:36.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:35 smithi145 bash[79878]: cluster 2024-03-07T03:35:34.597322+0000 mon.a (mon.0) 1574 : cluster 0 osdmap e909: 8 total, 8 up, 8 in 2024-03-07T03:35:36.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:35 smithi145 bash[82329]: cluster 2024-03-07T03:35:34.597322+0000 mon.a (mon.0) 1574 : cluster 0 osdmap e909: 8 total, 8 up, 8 in 2024-03-07T03:35:36.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:35 smithi189 bash[66969]: cluster 2024-03-07T03:35:34.597322+0000 mon.a (mon.0) 1574 : cluster 0 osdmap e909: 8 total, 8 up, 8 in 2024-03-07T03:35:37.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:36 smithi145 bash[79878]: cluster 2024-03-07T03:35:35.345980+0000 mgr.y (mgr.34107) 712 : cluster 0 pgmap v1246: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.1 MiB/s wr, 51 op/s 2024-03-07T03:35:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:36 smithi145 bash[82329]: cluster 2024-03-07T03:35:35.345980+0000 mgr.y (mgr.34107) 712 : cluster 0 pgmap v1246: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.1 MiB/s wr, 51 op/s 2024-03-07T03:35:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:36 smithi189 bash[66969]: cluster 2024-03-07T03:35:35.345980+0000 mgr.y (mgr.34107) 712 : cluster 0 pgmap v1246: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.1 MiB/s wr, 51 op/s 2024-03-07T03:35:39.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:38 smithi145 bash[82329]: cluster 2024-03-07T03:35:37.346984+0000 mgr.y (mgr.34107) 713 : cluster 0 pgmap v1247: 105 pgs: 105 active+clean; 355 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 57 op/s 2024-03-07T03:35:39.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:38 smithi145 bash[79878]: cluster 2024-03-07T03:35:37.346984+0000 mgr.y (mgr.34107) 713 : cluster 0 pgmap v1247: 105 pgs: 105 active+clean; 355 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 57 op/s 2024-03-07T03:35:39.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:38 smithi189 bash[66969]: cluster 2024-03-07T03:35:37.346984+0000 mgr.y (mgr.34107) 713 : cluster 0 pgmap v1247: 105 pgs: 105 active+clean; 355 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 57 op/s 2024-03-07T03:35:41.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:40 smithi145 bash[79878]: cluster 2024-03-07T03:35:39.347614+0000 mgr.y (mgr.34107) 714 : cluster 0 pgmap v1248: 105 pgs: 105 active+clean; 368 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 480 KiB/s rd, 2.9 MiB/s wr, 57 op/s 2024-03-07T03:35:41.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:40 smithi145 bash[82329]: cluster 2024-03-07T03:35:39.347614+0000 mgr.y (mgr.34107) 714 : cluster 0 pgmap v1248: 105 pgs: 105 active+clean; 368 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 480 KiB/s rd, 2.9 MiB/s wr, 57 op/s 2024-03-07T03:35:41.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:40 smithi189 bash[66969]: cluster 2024-03-07T03:35:39.347614+0000 mgr.y (mgr.34107) 714 : cluster 0 pgmap v1248: 105 pgs: 105 active+clean; 368 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 480 KiB/s rd, 2.9 MiB/s wr, 57 op/s 2024-03-07T03:35:41.343 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/2.ZeroOverlap (10889 ms) 2024-03-07T03:35:41.629 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/2 (82297 ms total) 2024-03-07T03:35:41.630 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:35:41.630 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/3, where TypeParam = LUKS1UnderLUKS1 2024-03-07T03:35:41.702 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/3.Simple 2024-03-07T03:35:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:35:42 smithi189 bash[59814]: ts=2024-03-07T03:35:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:35:43.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:35:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:35:42] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.43.0" 2024-03-07T03:35:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:42 smithi145 bash[79878]: cluster 2024-03-07T03:35:41.348593+0000 mgr.y (mgr.34107) 715 : cluster 0 pgmap v1249: 105 pgs: 105 active+clean; 372 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 952 KiB/s rd, 3.3 MiB/s wr, 62 op/s 2024-03-07T03:35:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:42 smithi145 bash[79878]: cluster 2024-03-07T03:35:41.629551+0000 mon.a (mon.0) 1575 : cluster 0 osdmap e910: 8 total, 8 up, 8 in 2024-03-07T03:35:43.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:42 smithi145 bash[82329]: cluster 2024-03-07T03:35:41.348593+0000 mgr.y (mgr.34107) 715 : cluster 0 pgmap v1249: 105 pgs: 105 active+clean; 372 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 952 KiB/s rd, 3.3 MiB/s wr, 62 op/s 2024-03-07T03:35:43.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:42 smithi145 bash[82329]: cluster 2024-03-07T03:35:41.629551+0000 mon.a (mon.0) 1575 : cluster 0 osdmap e910: 8 total, 8 up, 8 in 2024-03-07T03:35:43.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:42 smithi189 bash[66969]: cluster 2024-03-07T03:35:41.348593+0000 mgr.y (mgr.34107) 715 : cluster 0 pgmap v1249: 105 pgs: 105 active+clean; 372 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 952 KiB/s rd, 3.3 MiB/s wr, 62 op/s 2024-03-07T03:35:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:42 smithi189 bash[66969]: cluster 2024-03-07T03:35:41.629551+0000 mon.a (mon.0) 1575 : cluster 0 osdmap e910: 8 total, 8 up, 8 in 2024-03-07T03:35:43.672 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:35:44.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:43 smithi145 bash[79878]: cluster 2024-03-07T03:35:42.664811+0000 mon.a (mon.0) 1576 : cluster 0 osdmap e911: 8 total, 8 up, 8 in 2024-03-07T03:35:44.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:43 smithi145 bash[79878]: audit 2024-03-07T03:35:42.666873+0000 mon.a (mon.0) 1577 : audit 1 from='client.? 172.21.15.145:0/4255056354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-39","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:35:44.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:43 smithi145 bash[82329]: cluster 2024-03-07T03:35:42.664811+0000 mon.a (mon.0) 1576 : cluster 0 osdmap e911: 8 total, 8 up, 8 in 2024-03-07T03:35:44.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:43 smithi145 bash[82329]: audit 2024-03-07T03:35:42.666873+0000 mon.a (mon.0) 1577 : audit 1 from='client.? 172.21.15.145:0/4255056354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-39","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:35:44.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:43 smithi189 bash[66969]: cluster 2024-03-07T03:35:42.664811+0000 mon.a (mon.0) 1576 : cluster 0 osdmap e911: 8 total, 8 up, 8 in 2024-03-07T03:35:44.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:43 smithi189 bash[66969]: audit 2024-03-07T03:35:42.666873+0000 mon.a (mon.0) 1577 : audit 1 from='client.? 172.21.15.145:0/4255056354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-39","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:35:45.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:44 smithi145 bash[79878]: cluster 2024-03-07T03:35:43.349450+0000 mgr.y (mgr.34107) 716 : cluster 0 pgmap v1252: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:35:45.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:44 smithi145 bash[79878]: cluster 2024-03-07T03:35:43.660336+0000 mon.a (mon.0) 1578 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:35:45.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:44 smithi145 bash[79878]: audit 2024-03-07T03:35:43.669860+0000 mon.a (mon.0) 1579 : audit 1 from='client.? 172.21.15.145:0/4255056354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-39","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:35:45.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:44 smithi145 bash[79878]: cluster 2024-03-07T03:35:43.678801+0000 mon.a (mon.0) 1580 : cluster 0 osdmap e912: 8 total, 8 up, 8 in 2024-03-07T03:35:45.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:44 smithi145 bash[82329]: cluster 2024-03-07T03:35:43.349450+0000 mgr.y (mgr.34107) 716 : cluster 0 pgmap v1252: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:35:45.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:44 smithi145 bash[82329]: cluster 2024-03-07T03:35:43.660336+0000 mon.a (mon.0) 1578 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:35:45.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:44 smithi145 bash[82329]: audit 2024-03-07T03:35:43.669860+0000 mon.a (mon.0) 1579 : audit 1 from='client.? 172.21.15.145:0/4255056354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-39","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:35:45.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:44 smithi145 bash[82329]: cluster 2024-03-07T03:35:43.678801+0000 mon.a (mon.0) 1580 : cluster 0 osdmap e912: 8 total, 8 up, 8 in 2024-03-07T03:35:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:44 smithi189 bash[66969]: cluster 2024-03-07T03:35:43.349450+0000 mgr.y (mgr.34107) 716 : cluster 0 pgmap v1252: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:35:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:44 smithi189 bash[66969]: cluster 2024-03-07T03:35:43.660336+0000 mon.a (mon.0) 1578 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:35:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:44 smithi189 bash[66969]: audit 2024-03-07T03:35:43.669860+0000 mon.a (mon.0) 1579 : audit 1 from='client.? 172.21.15.145:0/4255056354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-39","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:35:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:44 smithi189 bash[66969]: cluster 2024-03-07T03:35:43.678801+0000 mon.a (mon.0) 1580 : cluster 0 osdmap e912: 8 total, 8 up, 8 in 2024-03-07T03:35:46.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:45 smithi145 bash[82329]: cluster 2024-03-07T03:35:44.678433+0000 mon.a (mon.0) 1581 : cluster 0 osdmap e913: 8 total, 8 up, 8 in 2024-03-07T03:35:46.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:45 smithi145 bash[79878]: cluster 2024-03-07T03:35:44.678433+0000 mon.a (mon.0) 1581 : cluster 0 osdmap e913: 8 total, 8 up, 8 in 2024-03-07T03:35:46.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:45 smithi189 bash[66969]: cluster 2024-03-07T03:35:44.678433+0000 mon.a (mon.0) 1581 : cluster 0 osdmap e913: 8 total, 8 up, 8 in 2024-03-07T03:35:47.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:46 smithi145 bash[82329]: cluster 2024-03-07T03:35:45.350245+0000 mgr.y (mgr.34107) 717 : cluster 0 pgmap v1255: 105 pgs: 6 creating+activating, 4 creating+peering, 13 unknown, 82 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:35:47.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:46 smithi145 bash[82329]: cluster 2024-03-07T03:35:45.682446+0000 mon.a (mon.0) 1582 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:35:47.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:46 smithi145 bash[82329]: cluster 2024-03-07T03:35:45.696516+0000 mon.a (mon.0) 1583 : cluster 0 osdmap e914: 8 total, 8 up, 8 in 2024-03-07T03:35:47.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:46 smithi145 bash[79878]: cluster 2024-03-07T03:35:45.350245+0000 mgr.y (mgr.34107) 717 : cluster 0 pgmap v1255: 105 pgs: 6 creating+activating, 4 creating+peering, 13 unknown, 82 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:35:47.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:46 smithi145 bash[79878]: cluster 2024-03-07T03:35:45.682446+0000 mon.a (mon.0) 1582 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:35:47.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:46 smithi145 bash[79878]: cluster 2024-03-07T03:35:45.696516+0000 mon.a (mon.0) 1583 : cluster 0 osdmap e914: 8 total, 8 up, 8 in 2024-03-07T03:35:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:46 smithi189 bash[66969]: cluster 2024-03-07T03:35:45.350245+0000 mgr.y (mgr.34107) 717 : cluster 0 pgmap v1255: 105 pgs: 6 creating+activating, 4 creating+peering, 13 unknown, 82 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:35:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:46 smithi189 bash[66969]: cluster 2024-03-07T03:35:45.682446+0000 mon.a (mon.0) 1582 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:35:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:46 smithi189 bash[66969]: cluster 2024-03-07T03:35:45.696516+0000 mon.a (mon.0) 1583 : cluster 0 osdmap e914: 8 total, 8 up, 8 in 2024-03-07T03:35:49.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:48 smithi145 bash[82329]: cluster 2024-03-07T03:35:47.351234+0000 mgr.y (mgr.34107) 718 : cluster 0 pgmap v1257: 105 pgs: 6 creating+activating, 4 creating+peering, 95 active+clean; 582 KiB data, 588 MiB used, 715 GiB / 715 GiB avail; 655 B/s rd, 655 B/s wr, 2 op/s 2024-03-07T03:35:49.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:48 smithi145 bash[82329]: audit 2024-03-07T03:35:47.662548+0000 mon.a (mon.0) 1584 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:48 smithi145 bash[79878]: cluster 2024-03-07T03:35:47.351234+0000 mgr.y (mgr.34107) 718 : cluster 0 pgmap v1257: 105 pgs: 6 creating+activating, 4 creating+peering, 95 active+clean; 582 KiB data, 588 MiB used, 715 GiB / 715 GiB avail; 655 B/s rd, 655 B/s wr, 2 op/s 2024-03-07T03:35:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:48 smithi145 bash[79878]: audit 2024-03-07T03:35:47.662548+0000 mon.a (mon.0) 1584 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:48 smithi189 bash[66969]: cluster 2024-03-07T03:35:47.351234+0000 mgr.y (mgr.34107) 718 : cluster 0 pgmap v1257: 105 pgs: 6 creating+activating, 4 creating+peering, 95 active+clean; 582 KiB data, 588 MiB used, 715 GiB / 715 GiB avail; 655 B/s rd, 655 B/s wr, 2 op/s 2024-03-07T03:35:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:48 smithi189 bash[66969]: audit 2024-03-07T03:35:47.662548+0000 mon.a (mon.0) 1584 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:51.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:50 smithi145 bash[79878]: cluster 2024-03-07T03:35:49.352254+0000 mgr.y (mgr.34107) 719 : cluster 0 pgmap v1258: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-03-07T03:35:51.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:50 smithi145 bash[82329]: cluster 2024-03-07T03:35:49.352254+0000 mgr.y (mgr.34107) 719 : cluster 0 pgmap v1258: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-03-07T03:35:51.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:50 smithi189 bash[66969]: cluster 2024-03-07T03:35:49.352254+0000 mgr.y (mgr.34107) 719 : cluster 0 pgmap v1258: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-03-07T03:35:52.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:51 smithi145 bash[79878]: cluster 2024-03-07T03:35:50.664607+0000 mon.a (mon.0) 1585 : cluster 0 osdmap e915: 8 total, 8 up, 8 in 2024-03-07T03:35:52.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:51 smithi145 bash[82329]: cluster 2024-03-07T03:35:50.664607+0000 mon.a (mon.0) 1585 : cluster 0 osdmap e915: 8 total, 8 up, 8 in 2024-03-07T03:35:52.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:51 smithi189 bash[66969]: cluster 2024-03-07T03:35:50.664607+0000 mon.a (mon.0) 1585 : cluster 0 osdmap e915: 8 total, 8 up, 8 in 2024-03-07T03:35:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:35:52 smithi189 bash[59814]: ts=2024-03-07T03:35:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:35:53.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:35:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:35:52] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.43.0" 2024-03-07T03:35:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:52 smithi145 bash[79878]: cluster 2024-03-07T03:35:51.353108+0000 mgr.y (mgr.34107) 720 : cluster 0 pgmap v1260: 105 pgs: 105 active+clean; 17 MiB data, 603 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-03-07T03:35:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:52 smithi145 bash[79878]: cluster 2024-03-07T03:35:52.652348+0000 mon.a (mon.0) 1586 : cluster 0 osdmap e916: 8 total, 8 up, 8 in 2024-03-07T03:35:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:52 smithi145 bash[82329]: cluster 2024-03-07T03:35:51.353108+0000 mgr.y (mgr.34107) 720 : cluster 0 pgmap v1260: 105 pgs: 105 active+clean; 17 MiB data, 603 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-03-07T03:35:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:52 smithi145 bash[82329]: cluster 2024-03-07T03:35:52.652348+0000 mon.a (mon.0) 1586 : cluster 0 osdmap e916: 8 total, 8 up, 8 in 2024-03-07T03:35:53.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:52 smithi189 bash[66969]: cluster 2024-03-07T03:35:51.353108+0000 mgr.y (mgr.34107) 720 : cluster 0 pgmap v1260: 105 pgs: 105 active+clean; 17 MiB data, 603 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-03-07T03:35:53.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:52 smithi189 bash[66969]: cluster 2024-03-07T03:35:52.652348+0000 mon.a (mon.0) 1586 : cluster 0 osdmap e916: 8 total, 8 up, 8 in 2024-03-07T03:35:55.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:54 smithi145 bash[79878]: cluster 2024-03-07T03:35:53.353781+0000 mgr.y (mgr.34107) 721 : cluster 0 pgmap v1262: 105 pgs: 105 active+clean; 21 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 2.6 MiB/s wr, 24 op/s 2024-03-07T03:35:55.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:54 smithi145 bash[79878]: audit 2024-03-07T03:35:54.556180+0000 mon.a (mon.0) 1587 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:55.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:54 smithi145 bash[82329]: cluster 2024-03-07T03:35:53.353781+0000 mgr.y (mgr.34107) 721 : cluster 0 pgmap v1262: 105 pgs: 105 active+clean; 21 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 2.6 MiB/s wr, 24 op/s 2024-03-07T03:35:55.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:54 smithi145 bash[82329]: audit 2024-03-07T03:35:54.556180+0000 mon.a (mon.0) 1587 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:55.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:54 smithi189 bash[66969]: cluster 2024-03-07T03:35:53.353781+0000 mgr.y (mgr.34107) 721 : cluster 0 pgmap v1262: 105 pgs: 105 active+clean; 21 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 2.6 MiB/s wr, 24 op/s 2024-03-07T03:35:55.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:54 smithi189 bash[66969]: audit 2024-03-07T03:35:54.556180+0000 mon.a (mon.0) 1587 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:35:57.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:56 smithi145 bash[79878]: cluster 2024-03-07T03:35:55.354888+0000 mgr.y (mgr.34107) 722 : cluster 0 pgmap v1263: 105 pgs: 105 active+clean; 30 MiB data, 669 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 3.6 MiB/s wr, 46 op/s 2024-03-07T03:35:57.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:56 smithi145 bash[82329]: cluster 2024-03-07T03:35:55.354888+0000 mgr.y (mgr.34107) 722 : cluster 0 pgmap v1263: 105 pgs: 105 active+clean; 30 MiB data, 669 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 3.6 MiB/s wr, 46 op/s 2024-03-07T03:35:57.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:56 smithi189 bash[66969]: cluster 2024-03-07T03:35:55.354888+0000 mgr.y (mgr.34107) 722 : cluster 0 pgmap v1263: 105 pgs: 105 active+clean; 30 MiB data, 669 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 3.6 MiB/s wr, 46 op/s 2024-03-07T03:35:59.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:35:58 smithi145 bash[79878]: cluster 2024-03-07T03:35:57.355959+0000 mgr.y (mgr.34107) 723 : cluster 0 pgmap v1264: 105 pgs: 105 active+clean; 34 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 4.1 MiB/s wr, 45 op/s 2024-03-07T03:35:59.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:35:58 smithi145 bash[82329]: cluster 2024-03-07T03:35:57.355959+0000 mgr.y (mgr.34107) 723 : cluster 0 pgmap v1264: 105 pgs: 105 active+clean; 34 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 4.1 MiB/s wr, 45 op/s 2024-03-07T03:35:59.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:35:58 smithi189 bash[66969]: cluster 2024-03-07T03:35:57.355959+0000 mgr.y (mgr.34107) 723 : cluster 0 pgmap v1264: 105 pgs: 105 active+clean; 34 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 4.1 MiB/s wr, 45 op/s 2024-03-07T03:36:01.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:00 smithi145 bash[79878]: cluster 2024-03-07T03:35:59.356905+0000 mgr.y (mgr.34107) 724 : cluster 0 pgmap v1265: 105 pgs: 105 active+clean; 34 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.9 MiB/s wr, 47 op/s 2024-03-07T03:36:01.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:00 smithi145 bash[82329]: cluster 2024-03-07T03:35:59.356905+0000 mgr.y (mgr.34107) 724 : cluster 0 pgmap v1265: 105 pgs: 105 active+clean; 34 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.9 MiB/s wr, 47 op/s 2024-03-07T03:36:01.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:00 smithi189 bash[66969]: cluster 2024-03-07T03:35:59.356905+0000 mgr.y (mgr.34107) 724 : cluster 0 pgmap v1265: 105 pgs: 105 active+clean; 34 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.9 MiB/s wr, 47 op/s 2024-03-07T03:36:01.251 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/3.Simple (19549 ms) 2024-03-07T03:36:01.251 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/3.Grow 2024-03-07T03:36:01.251 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:36:02.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:36:02 smithi189 bash[59814]: ts=2024-03-07T03:36:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:36:03.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:36:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:36:02] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2024-03-07T03:36:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:02 smithi145 bash[79878]: cluster 2024-03-07T03:36:01.357854+0000 mgr.y (mgr.34107) 725 : cluster 0 pgmap v1266: 105 pgs: 105 active+clean; 51 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 3.8 MiB/s wr, 43 op/s 2024-03-07T03:36:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:02 smithi145 bash[79878]: audit 2024-03-07T03:36:02.662892+0000 mon.a (mon.0) 1588 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:02 smithi145 bash[82329]: cluster 2024-03-07T03:36:01.357854+0000 mgr.y (mgr.34107) 725 : cluster 0 pgmap v1266: 105 pgs: 105 active+clean; 51 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 3.8 MiB/s wr, 43 op/s 2024-03-07T03:36:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:02 smithi145 bash[82329]: audit 2024-03-07T03:36:02.662892+0000 mon.a (mon.0) 1588 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:03.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:02 smithi189 bash[66969]: cluster 2024-03-07T03:36:01.357854+0000 mgr.y (mgr.34107) 725 : cluster 0 pgmap v1266: 105 pgs: 105 active+clean; 51 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 3.8 MiB/s wr, 43 op/s 2024-03-07T03:36:03.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:02 smithi189 bash[66969]: audit 2024-03-07T03:36:02.662892+0000 mon.a (mon.0) 1588 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:05.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:04 smithi145 bash[79878]: cluster 2024-03-07T03:36:03.358780+0000 mgr.y (mgr.34107) 726 : cluster 0 pgmap v1267: 105 pgs: 105 active+clean; 55 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.5 MiB/s wr, 34 op/s 2024-03-07T03:36:05.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:04 smithi145 bash[82329]: cluster 2024-03-07T03:36:03.358780+0000 mgr.y (mgr.34107) 726 : cluster 0 pgmap v1267: 105 pgs: 105 active+clean; 55 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.5 MiB/s wr, 34 op/s 2024-03-07T03:36:05.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:04 smithi189 bash[66969]: cluster 2024-03-07T03:36:03.358780+0000 mgr.y (mgr.34107) 726 : cluster 0 pgmap v1267: 105 pgs: 105 active+clean; 55 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.5 MiB/s wr, 34 op/s 2024-03-07T03:36:07.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:06 smithi145 bash[79878]: cluster 2024-03-07T03:36:05.360000+0000 mgr.y (mgr.34107) 727 : cluster 0 pgmap v1268: 105 pgs: 105 active+clean; 63 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 3.8 MiB/s wr, 39 op/s 2024-03-07T03:36:07.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:06 smithi145 bash[79878]: cluster 2024-03-07T03:36:05.719173+0000 mon.a (mon.0) 1589 : cluster 0 osdmap e917: 8 total, 8 up, 8 in 2024-03-07T03:36:07.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:06 smithi145 bash[79878]: audit 2024-03-07T03:36:06.582408+0000 mon.a (mon.0) 1590 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:36:07.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:06 smithi145 bash[82329]: cluster 2024-03-07T03:36:05.360000+0000 mgr.y (mgr.34107) 727 : cluster 0 pgmap v1268: 105 pgs: 105 active+clean; 63 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 3.8 MiB/s wr, 39 op/s 2024-03-07T03:36:07.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:06 smithi145 bash[82329]: cluster 2024-03-07T03:36:05.719173+0000 mon.a (mon.0) 1589 : cluster 0 osdmap e917: 8 total, 8 up, 8 in 2024-03-07T03:36:07.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:06 smithi145 bash[82329]: audit 2024-03-07T03:36:06.582408+0000 mon.a (mon.0) 1590 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:36:07.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:06 smithi189 bash[66969]: cluster 2024-03-07T03:36:05.360000+0000 mgr.y (mgr.34107) 727 : cluster 0 pgmap v1268: 105 pgs: 105 active+clean; 63 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 3.8 MiB/s wr, 39 op/s 2024-03-07T03:36:07.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:06 smithi189 bash[66969]: cluster 2024-03-07T03:36:05.719173+0000 mon.a (mon.0) 1589 : cluster 0 osdmap e917: 8 total, 8 up, 8 in 2024-03-07T03:36:07.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:06 smithi189 bash[66969]: audit 2024-03-07T03:36:06.582408+0000 mon.a (mon.0) 1590 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:36:09.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:08 smithi145 bash[82329]: cluster 2024-03-07T03:36:07.361003+0000 mgr.y (mgr.34107) 728 : cluster 0 pgmap v1270: 105 pgs: 105 active+clean; 67 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 45 op/s 2024-03-07T03:36:09.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:08 smithi145 bash[79878]: cluster 2024-03-07T03:36:07.361003+0000 mgr.y (mgr.34107) 728 : cluster 0 pgmap v1270: 105 pgs: 105 active+clean; 67 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 45 op/s 2024-03-07T03:36:09.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:08 smithi189 bash[66969]: cluster 2024-03-07T03:36:07.361003+0000 mgr.y (mgr.34107) 728 : cluster 0 pgmap v1270: 105 pgs: 105 active+clean; 67 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 45 op/s 2024-03-07T03:36:11.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:10 smithi145 bash[82329]: cluster 2024-03-07T03:36:09.361898+0000 mgr.y (mgr.34107) 729 : cluster 0 pgmap v1271: 105 pgs: 105 active+clean; 76 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 4.6 MiB/s wr, 47 op/s 2024-03-07T03:36:11.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:10 smithi145 bash[79878]: cluster 2024-03-07T03:36:09.361898+0000 mgr.y (mgr.34107) 729 : cluster 0 pgmap v1271: 105 pgs: 105 active+clean; 76 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 4.6 MiB/s wr, 47 op/s 2024-03-07T03:36:11.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:10 smithi189 bash[66969]: cluster 2024-03-07T03:36:09.361898+0000 mgr.y (mgr.34107) 729 : cluster 0 pgmap v1271: 105 pgs: 105 active+clean; 76 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 4.6 MiB/s wr, 47 op/s 2024-03-07T03:36:12.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:36:12 smithi189 bash[59814]: ts=2024-03-07T03:36:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:36:13.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:36:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:36:12] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2024-03-07T03:36:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:12 smithi145 bash[79878]: cluster 2024-03-07T03:36:11.362764+0000 mgr.y (mgr.34107) 730 : cluster 0 pgmap v1272: 105 pgs: 105 active+clean; 92 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 4.1 MiB/s wr, 51 op/s 2024-03-07T03:36:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:12 smithi145 bash[79878]: audit 2024-03-07T03:36:12.318757+0000 mon.a (mon.0) 1591 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:12 smithi145 bash[79878]: audit 2024-03-07T03:36:12.327818+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:12 smithi145 bash[79878]: audit 2024-03-07T03:36:12.673967+0000 mon.a (mon.0) 1593 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:12 smithi145 bash[79878]: audit 2024-03-07T03:36:12.688165+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:12 smithi145 bash[82329]: cluster 2024-03-07T03:36:11.362764+0000 mgr.y (mgr.34107) 730 : cluster 0 pgmap v1272: 105 pgs: 105 active+clean; 92 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 4.1 MiB/s wr, 51 op/s 2024-03-07T03:36:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:12 smithi145 bash[82329]: audit 2024-03-07T03:36:12.318757+0000 mon.a (mon.0) 1591 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:12 smithi145 bash[82329]: audit 2024-03-07T03:36:12.327818+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:12 smithi145 bash[82329]: audit 2024-03-07T03:36:12.673967+0000 mon.a (mon.0) 1593 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:12 smithi145 bash[82329]: audit 2024-03-07T03:36:12.688165+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:12 smithi189 bash[66969]: cluster 2024-03-07T03:36:11.362764+0000 mgr.y (mgr.34107) 730 : cluster 0 pgmap v1272: 105 pgs: 105 active+clean; 92 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 4.1 MiB/s wr, 51 op/s 2024-03-07T03:36:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:12 smithi189 bash[66969]: audit 2024-03-07T03:36:12.318757+0000 mon.a (mon.0) 1591 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:12 smithi189 bash[66969]: audit 2024-03-07T03:36:12.327818+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:12 smithi189 bash[66969]: audit 2024-03-07T03:36:12.673967+0000 mon.a (mon.0) 1593 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:13.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:12 smithi189 bash[66969]: audit 2024-03-07T03:36:12.688165+0000 mon.a (mon.0) 1594 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:14.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:13 smithi145 bash[79878]: audit 2024-03-07T03:36:13.326304+0000 mon.a (mon.0) 1595 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:36:14.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:13 smithi145 bash[79878]: audit 2024-03-07T03:36:13.328076+0000 mon.a (mon.0) 1596 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:36:14.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:13 smithi145 bash[79878]: audit 2024-03-07T03:36:13.340013+0000 mon.a (mon.0) 1597 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:14.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:13 smithi145 bash[82329]: audit 2024-03-07T03:36:13.326304+0000 mon.a (mon.0) 1595 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:36:14.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:13 smithi145 bash[82329]: audit 2024-03-07T03:36:13.328076+0000 mon.a (mon.0) 1596 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:36:14.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:13 smithi145 bash[82329]: audit 2024-03-07T03:36:13.340013+0000 mon.a (mon.0) 1597 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:14.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:13 smithi189 bash[66969]: audit 2024-03-07T03:36:13.326304+0000 mon.a (mon.0) 1595 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:36:14.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:13 smithi189 bash[66969]: audit 2024-03-07T03:36:13.328076+0000 mon.a (mon.0) 1596 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:36:14.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:13 smithi189 bash[66969]: audit 2024-03-07T03:36:13.340013+0000 mon.a (mon.0) 1597 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:36:15.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:14 smithi145 bash[79878]: cluster 2024-03-07T03:36:13.363637+0000 mgr.y (mgr.34107) 731 : cluster 0 pgmap v1273: 105 pgs: 105 active+clean; 92 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 3.7 MiB/s wr, 51 op/s 2024-03-07T03:36:15.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:14 smithi145 bash[82329]: cluster 2024-03-07T03:36:13.363637+0000 mgr.y (mgr.34107) 731 : cluster 0 pgmap v1273: 105 pgs: 105 active+clean; 92 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 3.7 MiB/s wr, 51 op/s 2024-03-07T03:36:15.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:14 smithi189 bash[66969]: cluster 2024-03-07T03:36:13.363637+0000 mgr.y (mgr.34107) 731 : cluster 0 pgmap v1273: 105 pgs: 105 active+clean; 92 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 3.7 MiB/s wr, 51 op/s 2024-03-07T03:36:16.435 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/3.Grow (15184 ms) 2024-03-07T03:36:16.435 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/3.Shrink 2024-03-07T03:36:16.435 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:36:17.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:16 smithi145 bash[82329]: cluster 2024-03-07T03:36:15.364666+0000 mgr.y (mgr.34107) 732 : cluster 0 pgmap v1274: 105 pgs: 105 active+clean; 96 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.7 MiB/s wr, 45 op/s 2024-03-07T03:36:17.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:16 smithi145 bash[79878]: cluster 2024-03-07T03:36:15.364666+0000 mgr.y (mgr.34107) 732 : cluster 0 pgmap v1274: 105 pgs: 105 active+clean; 96 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.7 MiB/s wr, 45 op/s 2024-03-07T03:36:17.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:16 smithi189 bash[66969]: cluster 2024-03-07T03:36:15.364666+0000 mgr.y (mgr.34107) 732 : cluster 0 pgmap v1274: 105 pgs: 105 active+clean; 96 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.7 MiB/s wr, 45 op/s 2024-03-07T03:36:18.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:17 smithi145 bash[82329]: audit 2024-03-07T03:36:17.663369+0000 mon.a (mon.0) 1598 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:18.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:17 smithi145 bash[79878]: audit 2024-03-07T03:36:17.663369+0000 mon.a (mon.0) 1598 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:18.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:17 smithi189 bash[66969]: audit 2024-03-07T03:36:17.663369+0000 mon.a (mon.0) 1598 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:19.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:18 smithi145 bash[82329]: cluster 2024-03-07T03:36:17.365708+0000 mgr.y (mgr.34107) 733 : cluster 0 pgmap v1275: 105 pgs: 105 active+clean; 101 MiB data, 875 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 3.6 MiB/s wr, 46 op/s 2024-03-07T03:36:19.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:18 smithi145 bash[79878]: cluster 2024-03-07T03:36:17.365708+0000 mgr.y (mgr.34107) 733 : cluster 0 pgmap v1275: 105 pgs: 105 active+clean; 101 MiB data, 875 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 3.6 MiB/s wr, 46 op/s 2024-03-07T03:36:19.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:18 smithi189 bash[66969]: cluster 2024-03-07T03:36:17.365708+0000 mgr.y (mgr.34107) 733 : cluster 0 pgmap v1275: 105 pgs: 105 active+clean; 101 MiB data, 875 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 3.6 MiB/s wr, 46 op/s 2024-03-07T03:36:21.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:20 smithi145 bash[79878]: cluster 2024-03-07T03:36:19.366674+0000 mgr.y (mgr.34107) 734 : cluster 0 pgmap v1276: 105 pgs: 105 active+clean; 117 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:36:21.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:20 smithi145 bash[82329]: cluster 2024-03-07T03:36:19.366674+0000 mgr.y (mgr.34107) 734 : cluster 0 pgmap v1276: 105 pgs: 105 active+clean; 117 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:36:21.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:20 smithi189 bash[66969]: cluster 2024-03-07T03:36:19.366674+0000 mgr.y (mgr.34107) 734 : cluster 0 pgmap v1276: 105 pgs: 105 active+clean; 117 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:36:22.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:21 smithi145 bash[79878]: cluster 2024-03-07T03:36:20.771221+0000 mon.a (mon.0) 1599 : cluster 0 osdmap e918: 8 total, 8 up, 8 in 2024-03-07T03:36:22.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:21 smithi145 bash[82329]: cluster 2024-03-07T03:36:20.771221+0000 mon.a (mon.0) 1599 : cluster 0 osdmap e918: 8 total, 8 up, 8 in 2024-03-07T03:36:22.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:21 smithi189 bash[66969]: cluster 2024-03-07T03:36:20.771221+0000 mon.a (mon.0) 1599 : cluster 0 osdmap e918: 8 total, 8 up, 8 in 2024-03-07T03:36:22.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:36:22 smithi189 bash[59814]: ts=2024-03-07T03:36:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:36:23.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:22 smithi145 bash[82329]: cluster 2024-03-07T03:36:21.367437+0000 mgr.y (mgr.34107) 735 : cluster 0 pgmap v1278: 105 pgs: 105 active+clean; 130 MiB data, 917 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.2 MiB/s wr, 44 op/s 2024-03-07T03:36:23.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:22 smithi145 bash[79878]: cluster 2024-03-07T03:36:21.367437+0000 mgr.y (mgr.34107) 735 : cluster 0 pgmap v1278: 105 pgs: 105 active+clean; 130 MiB data, 917 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.2 MiB/s wr, 44 op/s 2024-03-07T03:36:23.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:36:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:36:22] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.43.0" 2024-03-07T03:36:23.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:22 smithi189 bash[66969]: cluster 2024-03-07T03:36:21.367437+0000 mgr.y (mgr.34107) 735 : cluster 0 pgmap v1278: 105 pgs: 105 active+clean; 130 MiB data, 917 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.2 MiB/s wr, 44 op/s 2024-03-07T03:36:25.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:24 smithi145 bash[79878]: cluster 2024-03-07T03:36:23.368281+0000 mgr.y (mgr.34107) 736 : cluster 0 pgmap v1279: 105 pgs: 105 active+clean; 134 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.6 MiB/s wr, 44 op/s 2024-03-07T03:36:25.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:24 smithi145 bash[82329]: cluster 2024-03-07T03:36:23.368281+0000 mgr.y (mgr.34107) 736 : cluster 0 pgmap v1279: 105 pgs: 105 active+clean; 134 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.6 MiB/s wr, 44 op/s 2024-03-07T03:36:25.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:24 smithi189 bash[66969]: cluster 2024-03-07T03:36:23.368281+0000 mgr.y (mgr.34107) 736 : cluster 0 pgmap v1279: 105 pgs: 105 active+clean; 134 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.6 MiB/s wr, 44 op/s 2024-03-07T03:36:27.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:26 smithi145 bash[82329]: cluster 2024-03-07T03:36:25.369268+0000 mgr.y (mgr.34107) 737 : cluster 0 pgmap v1280: 105 pgs: 105 active+clean; 146 MiB data, 986 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 5.0 MiB/s wr, 57 op/s 2024-03-07T03:36:27.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:26 smithi145 bash[79878]: cluster 2024-03-07T03:36:25.369268+0000 mgr.y (mgr.34107) 737 : cluster 0 pgmap v1280: 105 pgs: 105 active+clean; 146 MiB data, 986 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 5.0 MiB/s wr, 57 op/s 2024-03-07T03:36:27.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:26 smithi189 bash[66969]: cluster 2024-03-07T03:36:25.369268+0000 mgr.y (mgr.34107) 737 : cluster 0 pgmap v1280: 105 pgs: 105 active+clean; 146 MiB data, 986 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 5.0 MiB/s wr, 57 op/s 2024-03-07T03:36:29.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:28 smithi145 bash[79878]: cluster 2024-03-07T03:36:27.370227+0000 mgr.y (mgr.34107) 738 : cluster 0 pgmap v1281: 105 pgs: 105 active+clean; 146 MiB data, 1003 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 4.5 MiB/s wr, 52 op/s 2024-03-07T03:36:29.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:28 smithi145 bash[82329]: cluster 2024-03-07T03:36:27.370227+0000 mgr.y (mgr.34107) 738 : cluster 0 pgmap v1281: 105 pgs: 105 active+clean; 146 MiB data, 1003 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 4.5 MiB/s wr, 52 op/s 2024-03-07T03:36:29.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:28 smithi189 bash[66969]: cluster 2024-03-07T03:36:27.370227+0000 mgr.y (mgr.34107) 738 : cluster 0 pgmap v1281: 105 pgs: 105 active+clean; 146 MiB data, 1003 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 4.5 MiB/s wr, 52 op/s 2024-03-07T03:36:31.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:30 smithi145 bash[79878]: cluster 2024-03-07T03:36:29.371137+0000 mgr.y (mgr.34107) 739 : cluster 0 pgmap v1282: 105 pgs: 105 active+clean; 150 MiB data, 1006 MiB used, 714 GiB / 715 GiB avail; 574 KiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-03-07T03:36:31.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:30 smithi145 bash[82329]: cluster 2024-03-07T03:36:29.371137+0000 mgr.y (mgr.34107) 739 : cluster 0 pgmap v1282: 105 pgs: 105 active+clean; 150 MiB data, 1006 MiB used, 714 GiB / 715 GiB avail; 574 KiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-03-07T03:36:31.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:30 smithi189 bash[66969]: cluster 2024-03-07T03:36:29.371137+0000 mgr.y (mgr.34107) 739 : cluster 0 pgmap v1282: 105 pgs: 105 active+clean; 150 MiB data, 1006 MiB used, 714 GiB / 715 GiB avail; 574 KiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-03-07T03:36:31.462 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/3.Shrink (15026 ms) 2024-03-07T03:36:31.462 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/3.ShrinkToOne 2024-03-07T03:36:31.462 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:36:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:36:32 smithi189 bash[59814]: ts=2024-03-07T03:36:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:36:33.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:32 smithi145 bash[79878]: cluster 2024-03-07T03:36:31.372067+0000 mgr.y (mgr.34107) 740 : cluster 0 pgmap v1283: 105 pgs: 105 active+clean; 163 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.3 MiB/s wr, 51 op/s 2024-03-07T03:36:33.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:32 smithi145 bash[79878]: audit 2024-03-07T03:36:32.664090+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:33.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:36:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:36:32] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.43.0" 2024-03-07T03:36:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:32 smithi145 bash[82329]: cluster 2024-03-07T03:36:31.372067+0000 mgr.y (mgr.34107) 740 : cluster 0 pgmap v1283: 105 pgs: 105 active+clean; 163 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.3 MiB/s wr, 51 op/s 2024-03-07T03:36:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:32 smithi145 bash[82329]: audit 2024-03-07T03:36:32.664090+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:33.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:32 smithi189 bash[66969]: cluster 2024-03-07T03:36:31.372067+0000 mgr.y (mgr.34107) 740 : cluster 0 pgmap v1283: 105 pgs: 105 active+clean; 163 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 4.3 MiB/s wr, 51 op/s 2024-03-07T03:36:33.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:32 smithi189 bash[66969]: audit 2024-03-07T03:36:32.664090+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:35.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:34 smithi145 bash[79878]: cluster 2024-03-07T03:36:33.372889+0000 mgr.y (mgr.34107) 741 : cluster 0 pgmap v1284: 105 pgs: 105 active+clean; 167 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 3.1 MiB/s wr, 29 op/s 2024-03-07T03:36:35.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:34 smithi145 bash[82329]: cluster 2024-03-07T03:36:33.372889+0000 mgr.y (mgr.34107) 741 : cluster 0 pgmap v1284: 105 pgs: 105 active+clean; 167 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 3.1 MiB/s wr, 29 op/s 2024-03-07T03:36:35.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:34 smithi189 bash[66969]: cluster 2024-03-07T03:36:33.372889+0000 mgr.y (mgr.34107) 741 : cluster 0 pgmap v1284: 105 pgs: 105 active+clean; 167 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 3.1 MiB/s wr, 29 op/s 2024-03-07T03:36:37.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:36 smithi145 bash[79878]: cluster 2024-03-07T03:36:35.374099+0000 mgr.y (mgr.34107) 742 : cluster 0 pgmap v1285: 105 pgs: 105 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 3.8 MiB/s wr, 40 op/s 2024-03-07T03:36:37.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:36 smithi145 bash[79878]: cluster 2024-03-07T03:36:35.812930+0000 mon.a (mon.0) 1601 : cluster 0 osdmap e919: 8 total, 8 up, 8 in 2024-03-07T03:36:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:36 smithi145 bash[82329]: cluster 2024-03-07T03:36:35.374099+0000 mgr.y (mgr.34107) 742 : cluster 0 pgmap v1285: 105 pgs: 105 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 3.8 MiB/s wr, 40 op/s 2024-03-07T03:36:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:36 smithi145 bash[82329]: cluster 2024-03-07T03:36:35.812930+0000 mon.a (mon.0) 1601 : cluster 0 osdmap e919: 8 total, 8 up, 8 in 2024-03-07T03:36:37.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:36 smithi189 bash[66969]: cluster 2024-03-07T03:36:35.374099+0000 mgr.y (mgr.34107) 742 : cluster 0 pgmap v1285: 105 pgs: 105 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 3.8 MiB/s wr, 40 op/s 2024-03-07T03:36:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:36 smithi189 bash[66969]: cluster 2024-03-07T03:36:35.812930+0000 mon.a (mon.0) 1601 : cluster 0 osdmap e919: 8 total, 8 up, 8 in 2024-03-07T03:36:39.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:38 smithi145 bash[79878]: cluster 2024-03-07T03:36:37.375050+0000 mgr.y (mgr.34107) 743 : cluster 0 pgmap v1287: 105 pgs: 105 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 37 op/s 2024-03-07T03:36:39.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:38 smithi145 bash[82329]: cluster 2024-03-07T03:36:37.375050+0000 mgr.y (mgr.34107) 743 : cluster 0 pgmap v1287: 105 pgs: 105 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 37 op/s 2024-03-07T03:36:39.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:38 smithi189 bash[66969]: cluster 2024-03-07T03:36:37.375050+0000 mgr.y (mgr.34107) 743 : cluster 0 pgmap v1287: 105 pgs: 105 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 37 op/s 2024-03-07T03:36:41.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:40 smithi145 bash[79878]: cluster 2024-03-07T03:36:39.376003+0000 mgr.y (mgr.34107) 744 : cluster 0 pgmap v1288: 105 pgs: 105 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.4 MiB/s wr, 39 op/s 2024-03-07T03:36:41.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:40 smithi145 bash[82329]: cluster 2024-03-07T03:36:39.376003+0000 mgr.y (mgr.34107) 744 : cluster 0 pgmap v1288: 105 pgs: 105 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.4 MiB/s wr, 39 op/s 2024-03-07T03:36:41.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:40 smithi189 bash[66969]: cluster 2024-03-07T03:36:39.376003+0000 mgr.y (mgr.34107) 744 : cluster 0 pgmap v1288: 105 pgs: 105 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.4 MiB/s wr, 39 op/s 2024-03-07T03:36:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:36:42 smithi189 bash[59814]: ts=2024-03-07T03:36:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:36:43.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:36:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:36:42] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.43.0" 2024-03-07T03:36:43.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:42 smithi145 bash[82329]: cluster 2024-03-07T03:36:41.376938+0000 mgr.y (mgr.34107) 745 : cluster 0 pgmap v1289: 105 pgs: 105 active+clean; 204 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 4.5 MiB/s wr, 52 op/s 2024-03-07T03:36:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:42 smithi145 bash[79878]: cluster 2024-03-07T03:36:41.376938+0000 mgr.y (mgr.34107) 745 : cluster 0 pgmap v1289: 105 pgs: 105 active+clean; 204 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 4.5 MiB/s wr, 52 op/s 2024-03-07T03:36:43.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:42 smithi189 bash[66969]: cluster 2024-03-07T03:36:41.376938+0000 mgr.y (mgr.34107) 745 : cluster 0 pgmap v1289: 105 pgs: 105 active+clean; 204 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 4.5 MiB/s wr, 52 op/s 2024-03-07T03:36:45.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:44 smithi189 bash[66969]: cluster 2024-03-07T03:36:43.377769+0000 mgr.y (mgr.34107) 746 : cluster 0 pgmap v1290: 105 pgs: 105 active+clean; 208 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 4.5 MiB/s wr, 53 op/s 2024-03-07T03:36:45.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:44 smithi145 bash[79878]: cluster 2024-03-07T03:36:43.377769+0000 mgr.y (mgr.34107) 746 : cluster 0 pgmap v1290: 105 pgs: 105 active+clean; 208 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 4.5 MiB/s wr, 53 op/s 2024-03-07T03:36:45.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:44 smithi145 bash[82329]: cluster 2024-03-07T03:36:43.377769+0000 mgr.y (mgr.34107) 746 : cluster 0 pgmap v1290: 105 pgs: 105 active+clean; 208 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 4.5 MiB/s wr, 53 op/s 2024-03-07T03:36:46.246 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/3.ShrinkToOne (14785 ms) 2024-03-07T03:36:46.247 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/3.ShrinkToOneAfterSnapshot 2024-03-07T03:36:46.247 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:36:47.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:46 smithi189 bash[66969]: cluster 2024-03-07T03:36:45.378427+0000 mgr.y (mgr.34107) 747 : cluster 0 pgmap v1291: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-03-07T03:36:47.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:46 smithi145 bash[79878]: cluster 2024-03-07T03:36:45.378427+0000 mgr.y (mgr.34107) 747 : cluster 0 pgmap v1291: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-03-07T03:36:47.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:46 smithi145 bash[82329]: cluster 2024-03-07T03:36:45.378427+0000 mgr.y (mgr.34107) 747 : cluster 0 pgmap v1291: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-03-07T03:36:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:47 smithi189 bash[66969]: audit 2024-03-07T03:36:47.664163+0000 mon.a (mon.0) 1602 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:48.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:47 smithi145 bash[79878]: audit 2024-03-07T03:36:47.664163+0000 mon.a (mon.0) 1602 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:48.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:47 smithi145 bash[82329]: audit 2024-03-07T03:36:47.664163+0000 mon.a (mon.0) 1602 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:49.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:48 smithi189 bash[66969]: cluster 2024-03-07T03:36:47.378981+0000 mgr.y (mgr.34107) 748 : cluster 0 pgmap v1292: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.2 MiB/s wr, 39 op/s 2024-03-07T03:36:49.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:48 smithi145 bash[79878]: cluster 2024-03-07T03:36:47.378981+0000 mgr.y (mgr.34107) 748 : cluster 0 pgmap v1292: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.2 MiB/s wr, 39 op/s 2024-03-07T03:36:49.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:48 smithi145 bash[82329]: cluster 2024-03-07T03:36:47.378981+0000 mgr.y (mgr.34107) 748 : cluster 0 pgmap v1292: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.2 MiB/s wr, 39 op/s 2024-03-07T03:36:51.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:50 smithi145 bash[79878]: cluster 2024-03-07T03:36:49.379504+0000 mgr.y (mgr.34107) 749 : cluster 0 pgmap v1293: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 783 KiB/s rd, 2.3 MiB/s wr, 38 op/s 2024-03-07T03:36:51.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:50 smithi145 bash[82329]: cluster 2024-03-07T03:36:49.379504+0000 mgr.y (mgr.34107) 749 : cluster 0 pgmap v1293: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 783 KiB/s rd, 2.3 MiB/s wr, 38 op/s 2024-03-07T03:36:51.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:50 smithi189 bash[66969]: cluster 2024-03-07T03:36:49.379504+0000 mgr.y (mgr.34107) 749 : cluster 0 pgmap v1293: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 783 KiB/s rd, 2.3 MiB/s wr, 38 op/s 2024-03-07T03:36:52.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:51 smithi189 bash[66969]: cluster 2024-03-07T03:36:50.854465+0000 mon.a (mon.0) 1603 : cluster 0 osdmap e920: 8 total, 8 up, 8 in 2024-03-07T03:36:52.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:51 smithi145 bash[79878]: cluster 2024-03-07T03:36:50.854465+0000 mon.a (mon.0) 1603 : cluster 0 osdmap e920: 8 total, 8 up, 8 in 2024-03-07T03:36:52.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:51 smithi145 bash[82329]: cluster 2024-03-07T03:36:50.854465+0000 mon.a (mon.0) 1603 : cluster 0 osdmap e920: 8 total, 8 up, 8 in 2024-03-07T03:36:52.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:36:52 smithi189 bash[59814]: ts=2024-03-07T03:36:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:36:53.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:52 smithi145 bash[79878]: cluster 2024-03-07T03:36:51.380243+0000 mgr.y (mgr.34107) 750 : cluster 0 pgmap v1295: 105 pgs: 105 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 921 KiB/s rd, 2.0 MiB/s wr, 23 op/s 2024-03-07T03:36:53.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:36:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:36:52] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2024-03-07T03:36:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:52 smithi145 bash[82329]: cluster 2024-03-07T03:36:51.380243+0000 mgr.y (mgr.34107) 750 : cluster 0 pgmap v1295: 105 pgs: 105 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 921 KiB/s rd, 2.0 MiB/s wr, 23 op/s 2024-03-07T03:36:53.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:52 smithi189 bash[66969]: cluster 2024-03-07T03:36:51.380243+0000 mgr.y (mgr.34107) 750 : cluster 0 pgmap v1295: 105 pgs: 105 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 921 KiB/s rd, 2.0 MiB/s wr, 23 op/s 2024-03-07T03:36:55.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:54 smithi145 bash[79878]: cluster 2024-03-07T03:36:53.381065+0000 mgr.y (mgr.34107) 751 : cluster 0 pgmap v1296: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 896 KiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-03-07T03:36:55.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:54 smithi145 bash[82329]: cluster 2024-03-07T03:36:53.381065+0000 mgr.y (mgr.34107) 751 : cluster 0 pgmap v1296: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 896 KiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-03-07T03:36:55.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:54 smithi189 bash[66969]: cluster 2024-03-07T03:36:53.381065+0000 mgr.y (mgr.34107) 751 : cluster 0 pgmap v1296: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 896 KiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-03-07T03:36:57.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:56 smithi145 bash[82329]: cluster 2024-03-07T03:36:55.382558+0000 mgr.y (mgr.34107) 752 : cluster 0 pgmap v1297: 105 pgs: 105 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 871 KiB/s rd, 4.1 MiB/s wr, 33 op/s 2024-03-07T03:36:57.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:56 smithi145 bash[79878]: cluster 2024-03-07T03:36:55.382558+0000 mgr.y (mgr.34107) 752 : cluster 0 pgmap v1297: 105 pgs: 105 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 871 KiB/s rd, 4.1 MiB/s wr, 33 op/s 2024-03-07T03:36:57.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:56 smithi189 bash[66969]: cluster 2024-03-07T03:36:55.382558+0000 mgr.y (mgr.34107) 752 : cluster 0 pgmap v1297: 105 pgs: 105 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 871 KiB/s rd, 4.1 MiB/s wr, 33 op/s 2024-03-07T03:36:59.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:36:58 smithi145 bash[79878]: cluster 2024-03-07T03:36:57.383545+0000 mgr.y (mgr.34107) 753 : cluster 0 pgmap v1298: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 4.1 MiB/s wr, 53 op/s 2024-03-07T03:36:59.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:36:58 smithi145 bash[82329]: cluster 2024-03-07T03:36:57.383545+0000 mgr.y (mgr.34107) 753 : cluster 0 pgmap v1298: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 4.1 MiB/s wr, 53 op/s 2024-03-07T03:36:59.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:36:58 smithi189 bash[66969]: cluster 2024-03-07T03:36:57.383545+0000 mgr.y (mgr.34107) 753 : cluster 0 pgmap v1298: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 4.1 MiB/s wr, 53 op/s 2024-03-07T03:37:01.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:00 smithi145 bash[79878]: cluster 2024-03-07T03:36:59.384516+0000 mgr.y (mgr.34107) 754 : cluster 0 pgmap v1299: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-03-07T03:37:01.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:00 smithi145 bash[79878]: cluster 2024-03-07T03:36:59.886239+0000 mon.a (mon.0) 1604 : cluster 0 osdmap e921: 8 total, 8 up, 8 in 2024-03-07T03:37:01.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:00 smithi145 bash[82329]: cluster 2024-03-07T03:36:59.384516+0000 mgr.y (mgr.34107) 754 : cluster 0 pgmap v1299: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-03-07T03:37:01.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:00 smithi145 bash[82329]: cluster 2024-03-07T03:36:59.886239+0000 mon.a (mon.0) 1604 : cluster 0 osdmap e921: 8 total, 8 up, 8 in 2024-03-07T03:37:01.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:00 smithi189 bash[66969]: cluster 2024-03-07T03:36:59.384516+0000 mgr.y (mgr.34107) 754 : cluster 0 pgmap v1299: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-03-07T03:37:01.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:00 smithi189 bash[66969]: cluster 2024-03-07T03:36:59.886239+0000 mon.a (mon.0) 1604 : cluster 0 osdmap e921: 8 total, 8 up, 8 in 2024-03-07T03:37:02.415 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/3.ShrinkToOneAfterSnapshot (16168 ms) 2024-03-07T03:37:02.415 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/3.MinOverlap 2024-03-07T03:37:02.415 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:37:02.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:37:02 smithi189 bash[59814]: ts=2024-03-07T03:37:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:37:03.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:37:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:37:02] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.43.0" 2024-03-07T03:37:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:02 smithi145 bash[79878]: cluster 2024-03-07T03:37:01.385545+0000 mgr.y (mgr.34107) 755 : cluster 0 pgmap v1301: 105 pgs: 105 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 532 KiB/s rd, 4.9 MiB/s wr, 53 op/s 2024-03-07T03:37:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:02 smithi145 bash[79878]: audit 2024-03-07T03:37:02.664800+0000 mon.a (mon.0) 1605 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:02 smithi145 bash[82329]: cluster 2024-03-07T03:37:01.385545+0000 mgr.y (mgr.34107) 755 : cluster 0 pgmap v1301: 105 pgs: 105 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 532 KiB/s rd, 4.9 MiB/s wr, 53 op/s 2024-03-07T03:37:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:02 smithi145 bash[82329]: audit 2024-03-07T03:37:02.664800+0000 mon.a (mon.0) 1605 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:03.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:02 smithi189 bash[66969]: cluster 2024-03-07T03:37:01.385545+0000 mgr.y (mgr.34107) 755 : cluster 0 pgmap v1301: 105 pgs: 105 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 532 KiB/s rd, 4.9 MiB/s wr, 53 op/s 2024-03-07T03:37:03.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:02 smithi189 bash[66969]: audit 2024-03-07T03:37:02.664800+0000 mon.a (mon.0) 1605 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:05.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:04 smithi145 bash[82329]: cluster 2024-03-07T03:37:03.386422+0000 mgr.y (mgr.34107) 756 : cluster 0 pgmap v1302: 105 pgs: 105 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 947 KiB/s rd, 4.5 MiB/s wr, 60 op/s 2024-03-07T03:37:05.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:04 smithi145 bash[79878]: cluster 2024-03-07T03:37:03.386422+0000 mgr.y (mgr.34107) 756 : cluster 0 pgmap v1302: 105 pgs: 105 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 947 KiB/s rd, 4.5 MiB/s wr, 60 op/s 2024-03-07T03:37:05.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:04 smithi189 bash[66969]: cluster 2024-03-07T03:37:03.386422+0000 mgr.y (mgr.34107) 756 : cluster 0 pgmap v1302: 105 pgs: 105 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 947 KiB/s rd, 4.5 MiB/s wr, 60 op/s 2024-03-07T03:37:07.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:06 smithi145 bash[79878]: cluster 2024-03-07T03:37:05.387446+0000 mgr.y (mgr.34107) 757 : cluster 0 pgmap v1303: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.1 MiB/s wr, 59 op/s 2024-03-07T03:37:07.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:06 smithi145 bash[82329]: cluster 2024-03-07T03:37:05.387446+0000 mgr.y (mgr.34107) 757 : cluster 0 pgmap v1303: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.1 MiB/s wr, 59 op/s 2024-03-07T03:37:07.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:06 smithi189 bash[66969]: cluster 2024-03-07T03:37:05.387446+0000 mgr.y (mgr.34107) 757 : cluster 0 pgmap v1303: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.1 MiB/s wr, 59 op/s 2024-03-07T03:37:08.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:07 smithi145 bash[82329]: cluster 2024-03-07T03:37:06.915895+0000 mon.a (mon.0) 1606 : cluster 0 osdmap e922: 8 total, 8 up, 8 in 2024-03-07T03:37:08.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:07 smithi145 bash[79878]: cluster 2024-03-07T03:37:06.915895+0000 mon.a (mon.0) 1606 : cluster 0 osdmap e922: 8 total, 8 up, 8 in 2024-03-07T03:37:08.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:07 smithi189 bash[66969]: cluster 2024-03-07T03:37:06.915895+0000 mon.a (mon.0) 1606 : cluster 0 osdmap e922: 8 total, 8 up, 8 in 2024-03-07T03:37:09.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:08 smithi145 bash[82329]: cluster 2024-03-07T03:37:07.388011+0000 mgr.y (mgr.34107) 758 : cluster 0 pgmap v1305: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.1 MiB/s wr, 44 op/s 2024-03-07T03:37:09.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:08 smithi145 bash[79878]: cluster 2024-03-07T03:37:07.388011+0000 mgr.y (mgr.34107) 758 : cluster 0 pgmap v1305: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.1 MiB/s wr, 44 op/s 2024-03-07T03:37:09.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:08 smithi189 bash[66969]: cluster 2024-03-07T03:37:07.388011+0000 mgr.y (mgr.34107) 758 : cluster 0 pgmap v1305: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.1 MiB/s wr, 44 op/s 2024-03-07T03:37:11.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:10 smithi145 bash[79878]: cluster 2024-03-07T03:37:09.388936+0000 mgr.y (mgr.34107) 759 : cluster 0 pgmap v1306: 105 pgs: 105 active+clean; 327 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 8.6 MiB/s wr, 62 op/s 2024-03-07T03:37:11.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:10 smithi145 bash[82329]: cluster 2024-03-07T03:37:09.388936+0000 mgr.y (mgr.34107) 759 : cluster 0 pgmap v1306: 105 pgs: 105 active+clean; 327 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 8.6 MiB/s wr, 62 op/s 2024-03-07T03:37:11.329 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:10 smithi189 bash[66969]: cluster 2024-03-07T03:37:09.388936+0000 mgr.y (mgr.34107) 759 : cluster 0 pgmap v1306: 105 pgs: 105 active+clean; 327 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 8.6 MiB/s wr, 62 op/s 2024-03-07T03:37:12.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:37:12 smithi189 bash[59814]: ts=2024-03-07T03:37:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:37:13.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:37:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:37:12] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.43.0" 2024-03-07T03:37:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:12 smithi145 bash[82329]: cluster 2024-03-07T03:37:11.389881+0000 mgr.y (mgr.34107) 760 : cluster 0 pgmap v1307: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.6 MiB/s wr, 54 op/s 2024-03-07T03:37:13.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:12 smithi145 bash[79878]: cluster 2024-03-07T03:37:11.389881+0000 mgr.y (mgr.34107) 760 : cluster 0 pgmap v1307: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.6 MiB/s wr, 54 op/s 2024-03-07T03:37:13.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:12 smithi189 bash[66969]: cluster 2024-03-07T03:37:11.389881+0000 mgr.y (mgr.34107) 760 : cluster 0 pgmap v1307: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.6 MiB/s wr, 54 op/s 2024-03-07T03:37:14.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:13 smithi145 bash[79878]: audit 2024-03-07T03:37:13.431371+0000 mon.a (mon.0) 1607 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:37:14.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:13 smithi145 bash[82329]: audit 2024-03-07T03:37:13.431371+0000 mon.a (mon.0) 1607 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:37:14.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:13 smithi189 bash[66969]: audit 2024-03-07T03:37:13.431371+0000 mon.a (mon.0) 1607 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:37:15.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:14 smithi145 bash[82329]: cluster 2024-03-07T03:37:13.390716+0000 mgr.y (mgr.34107) 761 : cluster 0 pgmap v1308: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.6 MiB/s wr, 41 op/s 2024-03-07T03:37:15.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:14 smithi145 bash[79878]: cluster 2024-03-07T03:37:13.390716+0000 mgr.y (mgr.34107) 761 : cluster 0 pgmap v1308: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.6 MiB/s wr, 41 op/s 2024-03-07T03:37:15.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:14 smithi189 bash[66969]: cluster 2024-03-07T03:37:13.390716+0000 mgr.y (mgr.34107) 761 : cluster 0 pgmap v1308: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.6 MiB/s wr, 41 op/s 2024-03-07T03:37:17.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:16 smithi145 bash[79878]: cluster 2024-03-07T03:37:15.391717+0000 mgr.y (mgr.34107) 762 : cluster 0 pgmap v1309: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 2.9 MiB/s wr, 40 op/s 2024-03-07T03:37:17.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:16 smithi145 bash[82329]: cluster 2024-03-07T03:37:15.391717+0000 mgr.y (mgr.34107) 762 : cluster 0 pgmap v1309: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 2.9 MiB/s wr, 40 op/s 2024-03-07T03:37:17.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:16 smithi189 bash[66969]: cluster 2024-03-07T03:37:15.391717+0000 mgr.y (mgr.34107) 762 : cluster 0 pgmap v1309: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 2.9 MiB/s wr, 40 op/s 2024-03-07T03:37:17.509 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/3.MinOverlap (15094 ms) 2024-03-07T03:37:17.509 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/3.ZeroOverlap 2024-03-07T03:37:17.509 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:37:18.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:17 smithi145 bash[79878]: audit 2024-03-07T03:37:17.665133+0000 mon.a (mon.0) 1608 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:18.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:17 smithi145 bash[82329]: audit 2024-03-07T03:37:17.665133+0000 mon.a (mon.0) 1608 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:18.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:17 smithi189 bash[66969]: audit 2024-03-07T03:37:17.665133+0000 mon.a (mon.0) 1608 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:19.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:18 smithi145 bash[82329]: cluster 2024-03-07T03:37:17.392709+0000 mgr.y (mgr.34107) 763 : cluster 0 pgmap v1310: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 4.3 MiB/s wr, 40 op/s 2024-03-07T03:37:19.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:18 smithi145 bash[79878]: cluster 2024-03-07T03:37:17.392709+0000 mgr.y (mgr.34107) 763 : cluster 0 pgmap v1310: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 4.3 MiB/s wr, 40 op/s 2024-03-07T03:37:19.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:18 smithi189 bash[66969]: cluster 2024-03-07T03:37:17.392709+0000 mgr.y (mgr.34107) 763 : cluster 0 pgmap v1310: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 4.3 MiB/s wr, 40 op/s 2024-03-07T03:37:20.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[79878]: audit 2024-03-07T03:37:19.169862+0000 mon.a (mon.0) 1609 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[79878]: audit 2024-03-07T03:37:19.179214+0000 mon.a (mon.0) 1610 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[79878]: cluster 2024-03-07T03:37:19.393865+0000 mgr.y (mgr.34107) 764 : cluster 0 pgmap v1311: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.8 MiB/s wr, 36 op/s 2024-03-07T03:37:20.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[79878]: audit 2024-03-07T03:37:19.427104+0000 mon.a (mon.0) 1611 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[79878]: audit 2024-03-07T03:37:19.437224+0000 mon.a (mon.0) 1612 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[79878]: audit 2024-03-07T03:37:20.016102+0000 mon.a (mon.0) 1613 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:37:20.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[79878]: audit 2024-03-07T03:37:20.017457+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:37:20.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[79878]: audit 2024-03-07T03:37:20.024003+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[82329]: audit 2024-03-07T03:37:19.169862+0000 mon.a (mon.0) 1609 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[82329]: audit 2024-03-07T03:37:19.179214+0000 mon.a (mon.0) 1610 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[82329]: cluster 2024-03-07T03:37:19.393865+0000 mgr.y (mgr.34107) 764 : cluster 0 pgmap v1311: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.8 MiB/s wr, 36 op/s 2024-03-07T03:37:20.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[82329]: audit 2024-03-07T03:37:19.427104+0000 mon.a (mon.0) 1611 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[82329]: audit 2024-03-07T03:37:19.437224+0000 mon.a (mon.0) 1612 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[82329]: audit 2024-03-07T03:37:20.016102+0000 mon.a (mon.0) 1613 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:37:20.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[82329]: audit 2024-03-07T03:37:20.017457+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:37:20.560 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:20 smithi145 bash[82329]: audit 2024-03-07T03:37:20.024003+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.578 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:37:20 smithi189 bash[49584]: logger=cleanup t=2024-03-07T03:37:20.215227748Z level=info msg="Completed cleanup jobs" duration=66.161499ms 2024-03-07T03:37:20.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:20 smithi189 bash[66969]: audit 2024-03-07T03:37:19.169862+0000 mon.a (mon.0) 1609 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:20 smithi189 bash[66969]: audit 2024-03-07T03:37:19.179214+0000 mon.a (mon.0) 1610 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:20 smithi189 bash[66969]: cluster 2024-03-07T03:37:19.393865+0000 mgr.y (mgr.34107) 764 : cluster 0 pgmap v1311: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.8 MiB/s wr, 36 op/s 2024-03-07T03:37:20.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:20 smithi189 bash[66969]: audit 2024-03-07T03:37:19.427104+0000 mon.a (mon.0) 1611 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:20 smithi189 bash[66969]: audit 2024-03-07T03:37:19.437224+0000 mon.a (mon.0) 1612 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:20.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:20 smithi189 bash[66969]: audit 2024-03-07T03:37:20.016102+0000 mon.a (mon.0) 1613 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:37:20.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:20 smithi189 bash[66969]: audit 2024-03-07T03:37:20.017457+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:37:20.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:20 smithi189 bash[66969]: audit 2024-03-07T03:37:20.024003+0000 mon.a (mon.0) 1615 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:22 smithi189 bash[66969]: cluster 2024-03-07T03:37:21.394853+0000 mgr.y (mgr.34107) 765 : cluster 0 pgmap v1312: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.7 MiB/s wr, 29 op/s 2024-03-07T03:37:22.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:22 smithi189 bash[66969]: cluster 2024-03-07T03:37:21.428223+0000 mon.a (mon.0) 1616 : cluster 0 osdmap e923: 8 total, 8 up, 8 in 2024-03-07T03:37:22.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:37:22 smithi189 bash[59814]: ts=2024-03-07T03:37:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:37:22.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:22 smithi145 bash[79878]: cluster 2024-03-07T03:37:21.394853+0000 mgr.y (mgr.34107) 765 : cluster 0 pgmap v1312: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.7 MiB/s wr, 29 op/s 2024-03-07T03:37:22.751 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:22 smithi145 bash[79878]: cluster 2024-03-07T03:37:21.428223+0000 mon.a (mon.0) 1616 : cluster 0 osdmap e923: 8 total, 8 up, 8 in 2024-03-07T03:37:22.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:22 smithi145 bash[82329]: cluster 2024-03-07T03:37:21.394853+0000 mgr.y (mgr.34107) 765 : cluster 0 pgmap v1312: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.7 MiB/s wr, 29 op/s 2024-03-07T03:37:22.751 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:22 smithi145 bash[82329]: cluster 2024-03-07T03:37:21.428223+0000 mon.a (mon.0) 1616 : cluster 0 osdmap e923: 8 total, 8 up, 8 in 2024-03-07T03:37:23.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:37:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:37:22] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.43.0" 2024-03-07T03:37:24.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:24 smithi145 bash[79878]: cluster 2024-03-07T03:37:23.395667+0000 mgr.y (mgr.34107) 766 : cluster 0 pgmap v1314: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 27 op/s 2024-03-07T03:37:24.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:24 smithi145 bash[82329]: cluster 2024-03-07T03:37:23.395667+0000 mgr.y (mgr.34107) 766 : cluster 0 pgmap v1314: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 27 op/s 2024-03-07T03:37:24.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:24 smithi189 bash[66969]: cluster 2024-03-07T03:37:23.395667+0000 mgr.y (mgr.34107) 766 : cluster 0 pgmap v1314: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.6 MiB/s wr, 27 op/s 2024-03-07T03:37:26.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:26 smithi145 bash[79878]: cluster 2024-03-07T03:37:25.396824+0000 mgr.y (mgr.34107) 767 : cluster 0 pgmap v1315: 105 pgs: 105 active+clean; 364 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 34 op/s 2024-03-07T03:37:26.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:26 smithi145 bash[82329]: cluster 2024-03-07T03:37:25.396824+0000 mgr.y (mgr.34107) 767 : cluster 0 pgmap v1315: 105 pgs: 105 active+clean; 364 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 34 op/s 2024-03-07T03:37:26.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:26 smithi189 bash[66969]: cluster 2024-03-07T03:37:25.396824+0000 mgr.y (mgr.34107) 767 : cluster 0 pgmap v1315: 105 pgs: 105 active+clean; 364 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 34 op/s 2024-03-07T03:37:28.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:28 smithi145 bash[79878]: cluster 2024-03-07T03:37:27.397813+0000 mgr.y (mgr.34107) 768 : cluster 0 pgmap v1316: 105 pgs: 105 active+clean; 368 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 449 KiB/s rd, 2.9 MiB/s wr, 54 op/s 2024-03-07T03:37:28.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:28 smithi145 bash[82329]: cluster 2024-03-07T03:37:27.397813+0000 mgr.y (mgr.34107) 768 : cluster 0 pgmap v1316: 105 pgs: 105 active+clean; 368 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 449 KiB/s rd, 2.9 MiB/s wr, 54 op/s 2024-03-07T03:37:28.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:28 smithi189 bash[66969]: cluster 2024-03-07T03:37:27.397813+0000 mgr.y (mgr.34107) 768 : cluster 0 pgmap v1316: 105 pgs: 105 active+clean; 368 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 449 KiB/s rd, 2.9 MiB/s wr, 54 op/s 2024-03-07T03:37:30.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:30 smithi145 bash[82329]: cluster 2024-03-07T03:37:29.398756+0000 mgr.y (mgr.34107) 769 : cluster 0 pgmap v1317: 105 pgs: 105 active+clean; 368 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 449 KiB/s rd, 2.9 MiB/s wr, 53 op/s 2024-03-07T03:37:30.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:30 smithi145 bash[79878]: cluster 2024-03-07T03:37:29.398756+0000 mgr.y (mgr.34107) 769 : cluster 0 pgmap v1317: 105 pgs: 105 active+clean; 368 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 449 KiB/s rd, 2.9 MiB/s wr, 53 op/s 2024-03-07T03:37:30.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:30 smithi189 bash[66969]: cluster 2024-03-07T03:37:29.398756+0000 mgr.y (mgr.34107) 769 : cluster 0 pgmap v1317: 105 pgs: 105 active+clean; 368 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 449 KiB/s rd, 2.9 MiB/s wr, 53 op/s 2024-03-07T03:37:31.944 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/3.ZeroOverlap (14435 ms) 2024-03-07T03:37:32.456 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/3 (110242 ms total) 2024-03-07T03:37:32.456 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:37:32.456 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/4, where TypeParam = LUKS1UnderLUKS2 2024-03-07T03:37:32.494 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/4.Simple 2024-03-07T03:37:32.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:32 smithi189 bash[66969]: cluster 2024-03-07T03:37:31.399802+0000 mgr.y (mgr.34107) 770 : cluster 0 pgmap v1318: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 535 KiB/s rd, 3.7 MiB/s wr, 52 op/s 2024-03-07T03:37:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:37:32 smithi189 bash[59814]: ts=2024-03-07T03:37:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:37:32.750 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:32 smithi145 bash[79878]: cluster 2024-03-07T03:37:31.399802+0000 mgr.y (mgr.34107) 770 : cluster 0 pgmap v1318: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 535 KiB/s rd, 3.7 MiB/s wr, 52 op/s 2024-03-07T03:37:32.750 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:32 smithi145 bash[82329]: cluster 2024-03-07T03:37:31.399802+0000 mgr.y (mgr.34107) 770 : cluster 0 pgmap v1318: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 535 KiB/s rd, 3.7 MiB/s wr, 52 op/s 2024-03-07T03:37:33.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:37:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:37:32] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.43.0" 2024-03-07T03:37:33.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:33 smithi145 bash[79878]: cluster 2024-03-07T03:37:32.454953+0000 mon.a (mon.0) 1617 : cluster 0 osdmap e924: 8 total, 8 up, 8 in 2024-03-07T03:37:33.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:33 smithi145 bash[79878]: audit 2024-03-07T03:37:32.963437+0000 mon.a (mon.0) 1618 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:33.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:33 smithi145 bash[82329]: cluster 2024-03-07T03:37:32.454953+0000 mon.a (mon.0) 1617 : cluster 0 osdmap e924: 8 total, 8 up, 8 in 2024-03-07T03:37:33.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:33 smithi145 bash[82329]: audit 2024-03-07T03:37:32.963437+0000 mon.a (mon.0) 1618 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:33.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:33 smithi189 bash[66969]: cluster 2024-03-07T03:37:32.454953+0000 mon.a (mon.0) 1617 : cluster 0 osdmap e924: 8 total, 8 up, 8 in 2024-03-07T03:37:33.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:33 smithi189 bash[66969]: audit 2024-03-07T03:37:32.963437+0000 mon.a (mon.0) 1618 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:34.482 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:37:34.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:34 smithi145 bash[79878]: cluster 2024-03-07T03:37:33.400444+0000 mgr.y (mgr.34107) 771 : cluster 0 pgmap v1320: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:34.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:34 smithi145 bash[79878]: cluster 2024-03-07T03:37:33.477956+0000 mon.a (mon.0) 1619 : cluster 0 osdmap e925: 8 total, 8 up, 8 in 2024-03-07T03:37:34.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:34 smithi145 bash[79878]: audit 2024-03-07T03:37:33.482860+0000 mon.a (mon.0) 1620 : audit 1 from='client.? 172.21.15.145:0/3210444227' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-40","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:37:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:34 smithi145 bash[82329]: cluster 2024-03-07T03:37:33.400444+0000 mgr.y (mgr.34107) 771 : cluster 0 pgmap v1320: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:34 smithi145 bash[82329]: cluster 2024-03-07T03:37:33.477956+0000 mon.a (mon.0) 1619 : cluster 0 osdmap e925: 8 total, 8 up, 8 in 2024-03-07T03:37:34.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:34 smithi145 bash[82329]: audit 2024-03-07T03:37:33.482860+0000 mon.a (mon.0) 1620 : audit 1 from='client.? 172.21.15.145:0/3210444227' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-40","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:37:34.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:34 smithi189 bash[66969]: cluster 2024-03-07T03:37:33.400444+0000 mgr.y (mgr.34107) 771 : cluster 0 pgmap v1320: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:34.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:34 smithi189 bash[66969]: cluster 2024-03-07T03:37:33.477956+0000 mon.a (mon.0) 1619 : cluster 0 osdmap e925: 8 total, 8 up, 8 in 2024-03-07T03:37:34.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:34 smithi189 bash[66969]: audit 2024-03-07T03:37:33.482860+0000 mon.a (mon.0) 1620 : audit 1 from='client.? 172.21.15.145:0/3210444227' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-40","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:37:35.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:35 smithi145 bash[79878]: audit 2024-03-07T03:37:34.466094+0000 mon.a (mon.0) 1621 : audit 1 from='client.? 172.21.15.145:0/3210444227' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-40","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:37:35.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:35 smithi145 bash[79878]: cluster 2024-03-07T03:37:34.477534+0000 mon.a (mon.0) 1622 : cluster 0 osdmap e926: 8 total, 8 up, 8 in 2024-03-07T03:37:35.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:35 smithi145 bash[82329]: audit 2024-03-07T03:37:34.466094+0000 mon.a (mon.0) 1621 : audit 1 from='client.? 172.21.15.145:0/3210444227' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-40","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:37:35.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:35 smithi145 bash[82329]: cluster 2024-03-07T03:37:34.477534+0000 mon.a (mon.0) 1622 : cluster 0 osdmap e926: 8 total, 8 up, 8 in 2024-03-07T03:37:35.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:35 smithi189 bash[66969]: audit 2024-03-07T03:37:34.466094+0000 mon.a (mon.0) 1621 : audit 1 from='client.? 172.21.15.145:0/3210444227' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-40","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:37:35.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:35 smithi189 bash[66969]: cluster 2024-03-07T03:37:34.477534+0000 mon.a (mon.0) 1622 : cluster 0 osdmap e926: 8 total, 8 up, 8 in 2024-03-07T03:37:36.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:36 smithi145 bash[79878]: cluster 2024-03-07T03:37:35.401243+0000 mgr.y (mgr.34107) 772 : cluster 0 pgmap v1323: 105 pgs: 11 creating+peering, 16 unknown, 78 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:36.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:36 smithi145 bash[79878]: cluster 2024-03-07T03:37:35.498137+0000 mon.a (mon.0) 1623 : cluster 0 osdmap e927: 8 total, 8 up, 8 in 2024-03-07T03:37:36.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:36 smithi145 bash[82329]: cluster 2024-03-07T03:37:35.401243+0000 mgr.y (mgr.34107) 772 : cluster 0 pgmap v1323: 105 pgs: 11 creating+peering, 16 unknown, 78 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:36.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:36 smithi145 bash[82329]: cluster 2024-03-07T03:37:35.498137+0000 mon.a (mon.0) 1623 : cluster 0 osdmap e927: 8 total, 8 up, 8 in 2024-03-07T03:37:36.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:36 smithi189 bash[66969]: cluster 2024-03-07T03:37:35.401243+0000 mgr.y (mgr.34107) 772 : cluster 0 pgmap v1323: 105 pgs: 11 creating+peering, 16 unknown, 78 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:36.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:36 smithi189 bash[66969]: cluster 2024-03-07T03:37:35.498137+0000 mon.a (mon.0) 1623 : cluster 0 osdmap e927: 8 total, 8 up, 8 in 2024-03-07T03:37:37.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:37 smithi145 bash[79878]: cluster 2024-03-07T03:37:36.500729+0000 mon.a (mon.0) 1624 : cluster 0 osdmap e928: 8 total, 8 up, 8 in 2024-03-07T03:37:37.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:37 smithi145 bash[82329]: cluster 2024-03-07T03:37:36.500729+0000 mon.a (mon.0) 1624 : cluster 0 osdmap e928: 8 total, 8 up, 8 in 2024-03-07T03:37:37.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:37 smithi189 bash[66969]: cluster 2024-03-07T03:37:36.500729+0000 mon.a (mon.0) 1624 : cluster 0 osdmap e928: 8 total, 8 up, 8 in 2024-03-07T03:37:38.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:38 smithi145 bash[79878]: cluster 2024-03-07T03:37:37.402018+0000 mgr.y (mgr.34107) 773 : cluster 0 pgmap v1326: 105 pgs: 11 creating+peering, 94 active+clean; 582 KiB data, 563 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:37:38.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:38 smithi145 bash[82329]: cluster 2024-03-07T03:37:37.402018+0000 mgr.y (mgr.34107) 773 : cluster 0 pgmap v1326: 105 pgs: 11 creating+peering, 94 active+clean; 582 KiB data, 563 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:37:38.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:38 smithi189 bash[66969]: cluster 2024-03-07T03:37:37.402018+0000 mgr.y (mgr.34107) 773 : cluster 0 pgmap v1326: 105 pgs: 11 creating+peering, 94 active+clean; 582 KiB data, 563 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:37:40.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:40 smithi145 bash[82329]: cluster 2024-03-07T03:37:39.402960+0000 mgr.y (mgr.34107) 774 : cluster 0 pgmap v1327: 105 pgs: 105 active+clean; 582 KiB data, 562 MiB used, 715 GiB / 715 GiB avail; 517 B/s rd, 344 B/s wr, 1 op/s 2024-03-07T03:37:40.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:40 smithi145 bash[79878]: cluster 2024-03-07T03:37:39.402960+0000 mgr.y (mgr.34107) 774 : cluster 0 pgmap v1327: 105 pgs: 105 active+clean; 582 KiB data, 562 MiB used, 715 GiB / 715 GiB avail; 517 B/s rd, 344 B/s wr, 1 op/s 2024-03-07T03:37:40.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:40 smithi189 bash[66969]: cluster 2024-03-07T03:37:39.402960+0000 mgr.y (mgr.34107) 774 : cluster 0 pgmap v1327: 105 pgs: 105 active+clean; 582 KiB data, 562 MiB used, 715 GiB / 715 GiB avail; 517 B/s rd, 344 B/s wr, 1 op/s 2024-03-07T03:37:41.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:41 smithi145 bash[79878]: cluster 2024-03-07T03:37:40.540230+0000 mon.a (mon.0) 1625 : cluster 0 osdmap e929: 8 total, 8 up, 8 in 2024-03-07T03:37:41.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:41 smithi145 bash[82329]: cluster 2024-03-07T03:37:40.540230+0000 mon.a (mon.0) 1625 : cluster 0 osdmap e929: 8 total, 8 up, 8 in 2024-03-07T03:37:41.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:41 smithi189 bash[66969]: cluster 2024-03-07T03:37:40.540230+0000 mon.a (mon.0) 1625 : cluster 0 osdmap e929: 8 total, 8 up, 8 in 2024-03-07T03:37:42.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:42 smithi189 bash[66969]: cluster 2024-03-07T03:37:41.403800+0000 mgr.y (mgr.34107) 775 : cluster 0 pgmap v1329: 105 pgs: 105 active+clean; 13 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 2.0 MiB/s wr, 36 op/s 2024-03-07T03:37:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:37:42 smithi189 bash[59814]: ts=2024-03-07T03:37:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:37:42.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:42 smithi145 bash[79878]: cluster 2024-03-07T03:37:41.403800+0000 mgr.y (mgr.34107) 775 : cluster 0 pgmap v1329: 105 pgs: 105 active+clean; 13 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 2.0 MiB/s wr, 36 op/s 2024-03-07T03:37:42.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:37:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:37:42] "GET /metrics HTTP/1.1" 200 34146 "" "Prometheus/2.43.0" 2024-03-07T03:37:42.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:42 smithi145 bash[82329]: cluster 2024-03-07T03:37:41.403800+0000 mgr.y (mgr.34107) 775 : cluster 0 pgmap v1329: 105 pgs: 105 active+clean; 13 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 2.0 MiB/s wr, 36 op/s 2024-03-07T03:37:44.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:44 smithi145 bash[79878]: cluster 2024-03-07T03:37:43.404536+0000 mgr.y (mgr.34107) 776 : cluster 0 pgmap v1330: 105 pgs: 105 active+clean; 21 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 2.5 MiB/s wr, 28 op/s 2024-03-07T03:37:44.808 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:44 smithi145 bash[82329]: cluster 2024-03-07T03:37:43.404536+0000 mgr.y (mgr.34107) 776 : cluster 0 pgmap v1330: 105 pgs: 105 active+clean; 21 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 2.5 MiB/s wr, 28 op/s 2024-03-07T03:37:44.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:44 smithi189 bash[66969]: cluster 2024-03-07T03:37:43.404536+0000 mgr.y (mgr.34107) 776 : cluster 0 pgmap v1330: 105 pgs: 105 active+clean; 21 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 2.5 MiB/s wr, 28 op/s 2024-03-07T03:37:45.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:45 smithi189 bash[66969]: audit 2024-03-07T03:37:44.591557+0000 mon.a (mon.0) 1626 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:46.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:45 smithi145 bash[79878]: audit 2024-03-07T03:37:44.591557+0000 mon.a (mon.0) 1626 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:46.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:45 smithi145 bash[82329]: audit 2024-03-07T03:37:44.591557+0000 mon.a (mon.0) 1626 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:37:47.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:46 smithi145 bash[79878]: cluster 2024-03-07T03:37:45.405619+0000 mgr.y (mgr.34107) 777 : cluster 0 pgmap v1331: 105 pgs: 105 active+clean; 30 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 3.3 MiB/s wr, 41 op/s 2024-03-07T03:37:47.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:46 smithi145 bash[82329]: cluster 2024-03-07T03:37:45.405619+0000 mgr.y (mgr.34107) 777 : cluster 0 pgmap v1331: 105 pgs: 105 active+clean; 30 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 3.3 MiB/s wr, 41 op/s 2024-03-07T03:37:47.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:46 smithi189 bash[66969]: cluster 2024-03-07T03:37:45.405619+0000 mgr.y (mgr.34107) 777 : cluster 0 pgmap v1331: 105 pgs: 105 active+clean; 30 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 3.3 MiB/s wr, 41 op/s 2024-03-07T03:37:49.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:48 smithi145 bash[79878]: cluster 2024-03-07T03:37:47.406405+0000 mgr.y (mgr.34107) 778 : cluster 0 pgmap v1332: 105 pgs: 105 active+clean; 30 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 2.9 MiB/s wr, 40 op/s 2024-03-07T03:37:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:48 smithi145 bash[79878]: audit 2024-03-07T03:37:47.665887+0000 mon.a (mon.0) 1627 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:49.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:48 smithi145 bash[79878]: cluster 2024-03-07T03:37:47.970852+0000 mon.a (mon.0) 1628 : cluster 0 osdmap e930: 8 total, 8 up, 8 in 2024-03-07T03:37:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:48 smithi145 bash[82329]: cluster 2024-03-07T03:37:47.406405+0000 mgr.y (mgr.34107) 778 : cluster 0 pgmap v1332: 105 pgs: 105 active+clean; 30 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 2.9 MiB/s wr, 40 op/s 2024-03-07T03:37:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:48 smithi145 bash[82329]: audit 2024-03-07T03:37:47.665887+0000 mon.a (mon.0) 1627 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:49.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:48 smithi145 bash[82329]: cluster 2024-03-07T03:37:47.970852+0000 mon.a (mon.0) 1628 : cluster 0 osdmap e930: 8 total, 8 up, 8 in 2024-03-07T03:37:49.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:48 smithi189 bash[66969]: cluster 2024-03-07T03:37:47.406405+0000 mgr.y (mgr.34107) 778 : cluster 0 pgmap v1332: 105 pgs: 105 active+clean; 30 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 2.9 MiB/s wr, 40 op/s 2024-03-07T03:37:49.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:48 smithi189 bash[66969]: audit 2024-03-07T03:37:47.665887+0000 mon.a (mon.0) 1627 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:49.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:48 smithi189 bash[66969]: cluster 2024-03-07T03:37:47.970852+0000 mon.a (mon.0) 1628 : cluster 0 osdmap e930: 8 total, 8 up, 8 in 2024-03-07T03:37:51.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:50 smithi145 bash[79878]: cluster 2024-03-07T03:37:49.406906+0000 mgr.y (mgr.34107) 779 : cluster 0 pgmap v1334: 105 pgs: 105 active+clean; 30 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-03-07T03:37:51.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:50 smithi145 bash[82329]: cluster 2024-03-07T03:37:49.406906+0000 mgr.y (mgr.34107) 779 : cluster 0 pgmap v1334: 105 pgs: 105 active+clean; 30 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-03-07T03:37:51.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:50 smithi189 bash[66969]: cluster 2024-03-07T03:37:49.406906+0000 mgr.y (mgr.34107) 779 : cluster 0 pgmap v1334: 105 pgs: 105 active+clean; 30 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-03-07T03:37:52.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:37:52 smithi189 bash[59814]: ts=2024-03-07T03:37:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:37:53.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:52 smithi145 bash[79878]: cluster 2024-03-07T03:37:51.407423+0000 mgr.y (mgr.34107) 780 : cluster 0 pgmap v1335: 105 pgs: 105 active+clean; 34 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 2.1 MiB/s wr, 26 op/s 2024-03-07T03:37:53.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:52 smithi145 bash[82329]: cluster 2024-03-07T03:37:51.407423+0000 mgr.y (mgr.34107) 780 : cluster 0 pgmap v1335: 105 pgs: 105 active+clean; 34 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 2.1 MiB/s wr, 26 op/s 2024-03-07T03:37:53.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:37:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:37:52] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.43.0" 2024-03-07T03:37:53.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:52 smithi189 bash[66969]: cluster 2024-03-07T03:37:51.407423+0000 mgr.y (mgr.34107) 780 : cluster 0 pgmap v1335: 105 pgs: 105 active+clean; 34 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 2.1 MiB/s wr, 26 op/s 2024-03-07T03:37:55.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:54 smithi145 bash[79878]: cluster 2024-03-07T03:37:53.408013+0000 mgr.y (mgr.34107) 781 : cluster 0 pgmap v1336: 105 pgs: 105 active+clean; 38 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 1.7 MiB/s wr, 26 op/s 2024-03-07T03:37:55.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:54 smithi145 bash[82329]: cluster 2024-03-07T03:37:53.408013+0000 mgr.y (mgr.34107) 781 : cluster 0 pgmap v1336: 105 pgs: 105 active+clean; 38 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 1.7 MiB/s wr, 26 op/s 2024-03-07T03:37:55.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:54 smithi189 bash[66969]: cluster 2024-03-07T03:37:53.408013+0000 mgr.y (mgr.34107) 781 : cluster 0 pgmap v1336: 105 pgs: 105 active+clean; 38 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 1.7 MiB/s wr, 26 op/s 2024-03-07T03:37:55.087 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/4.Simple (22593 ms) 2024-03-07T03:37:55.088 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/4.Grow 2024-03-07T03:37:55.088 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:37:57.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:56 smithi145 bash[82329]: cluster 2024-03-07T03:37:55.408605+0000 mgr.y (mgr.34107) 782 : cluster 0 pgmap v1337: 105 pgs: 105 active+clean; 46 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2024-03-07T03:37:57.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:56 smithi145 bash[79878]: cluster 2024-03-07T03:37:55.408605+0000 mgr.y (mgr.34107) 782 : cluster 0 pgmap v1337: 105 pgs: 105 active+clean; 46 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2024-03-07T03:37:57.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:56 smithi189 bash[66969]: cluster 2024-03-07T03:37:55.408605+0000 mgr.y (mgr.34107) 782 : cluster 0 pgmap v1337: 105 pgs: 105 active+clean; 46 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2024-03-07T03:37:59.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:37:58 smithi145 bash[82329]: cluster 2024-03-07T03:37:57.409591+0000 mgr.y (mgr.34107) 783 : cluster 0 pgmap v1338: 105 pgs: 105 active+clean; 71 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 31 op/s 2024-03-07T03:37:59.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:37:58 smithi145 bash[79878]: cluster 2024-03-07T03:37:57.409591+0000 mgr.y (mgr.34107) 783 : cluster 0 pgmap v1338: 105 pgs: 105 active+clean; 71 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 31 op/s 2024-03-07T03:37:59.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:37:58 smithi189 bash[66969]: cluster 2024-03-07T03:37:57.409591+0000 mgr.y (mgr.34107) 783 : cluster 0 pgmap v1338: 105 pgs: 105 active+clean; 71 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 31 op/s 2024-03-07T03:38:01.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:00 smithi145 bash[79878]: cluster 2024-03-07T03:37:59.410578+0000 mgr.y (mgr.34107) 784 : cluster 0 pgmap v1339: 105 pgs: 105 active+clean; 71 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 27 op/s 2024-03-07T03:38:01.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:00 smithi145 bash[79878]: cluster 2024-03-07T03:37:59.636759+0000 mon.a (mon.0) 1629 : cluster 0 osdmap e931: 8 total, 8 up, 8 in 2024-03-07T03:38:01.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:00 smithi145 bash[82329]: cluster 2024-03-07T03:37:59.410578+0000 mgr.y (mgr.34107) 784 : cluster 0 pgmap v1339: 105 pgs: 105 active+clean; 71 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 27 op/s 2024-03-07T03:38:01.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:00 smithi145 bash[82329]: cluster 2024-03-07T03:37:59.636759+0000 mon.a (mon.0) 1629 : cluster 0 osdmap e931: 8 total, 8 up, 8 in 2024-03-07T03:38:01.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:00 smithi189 bash[66969]: cluster 2024-03-07T03:37:59.410578+0000 mgr.y (mgr.34107) 784 : cluster 0 pgmap v1339: 105 pgs: 105 active+clean; 71 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 27 op/s 2024-03-07T03:38:01.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:00 smithi189 bash[66969]: cluster 2024-03-07T03:37:59.636759+0000 mon.a (mon.0) 1629 : cluster 0 osdmap e931: 8 total, 8 up, 8 in 2024-03-07T03:38:02.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:38:02 smithi189 bash[59814]: ts=2024-03-07T03:38:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:38:03.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:02 smithi145 bash[79878]: cluster 2024-03-07T03:38:01.411373+0000 mgr.y (mgr.34107) 785 : cluster 0 pgmap v1341: 105 pgs: 105 active+clean; 87 MiB data, 807 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.9 MiB/s wr, 52 op/s 2024-03-07T03:38:03.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:38:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:38:02] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.43.0" 2024-03-07T03:38:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:02 smithi145 bash[82329]: cluster 2024-03-07T03:38:01.411373+0000 mgr.y (mgr.34107) 785 : cluster 0 pgmap v1341: 105 pgs: 105 active+clean; 87 MiB data, 807 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.9 MiB/s wr, 52 op/s 2024-03-07T03:38:03.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:02 smithi189 bash[66969]: cluster 2024-03-07T03:38:01.411373+0000 mgr.y (mgr.34107) 785 : cluster 0 pgmap v1341: 105 pgs: 105 active+clean; 87 MiB data, 807 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.9 MiB/s wr, 52 op/s 2024-03-07T03:38:04.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:03 smithi145 bash[79878]: audit 2024-03-07T03:38:02.665767+0000 mon.a (mon.0) 1630 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:04.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:03 smithi145 bash[82329]: audit 2024-03-07T03:38:02.665767+0000 mon.a (mon.0) 1630 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:04.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:03 smithi189 bash[66969]: audit 2024-03-07T03:38:02.665767+0000 mon.a (mon.0) 1630 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:05.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:04 smithi145 bash[79878]: cluster 2024-03-07T03:38:03.412053+0000 mgr.y (mgr.34107) 786 : cluster 0 pgmap v1342: 105 pgs: 105 active+clean; 95 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 60 op/s 2024-03-07T03:38:05.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:04 smithi145 bash[82329]: cluster 2024-03-07T03:38:03.412053+0000 mgr.y (mgr.34107) 786 : cluster 0 pgmap v1342: 105 pgs: 105 active+clean; 95 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 60 op/s 2024-03-07T03:38:05.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:04 smithi189 bash[66969]: cluster 2024-03-07T03:38:03.412053+0000 mgr.y (mgr.34107) 786 : cluster 0 pgmap v1342: 105 pgs: 105 active+clean; 95 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 60 op/s 2024-03-07T03:38:07.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:06 smithi145 bash[79878]: cluster 2024-03-07T03:38:05.412681+0000 mgr.y (mgr.34107) 787 : cluster 0 pgmap v1343: 105 pgs: 105 active+clean; 100 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 6.2 MiB/s wr, 51 op/s 2024-03-07T03:38:07.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:06 smithi145 bash[82329]: cluster 2024-03-07T03:38:05.412681+0000 mgr.y (mgr.34107) 787 : cluster 0 pgmap v1343: 105 pgs: 105 active+clean; 100 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 6.2 MiB/s wr, 51 op/s 2024-03-07T03:38:07.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:06 smithi189 bash[66969]: cluster 2024-03-07T03:38:05.412681+0000 mgr.y (mgr.34107) 787 : cluster 0 pgmap v1343: 105 pgs: 105 active+clean; 100 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 6.2 MiB/s wr, 51 op/s 2024-03-07T03:38:09.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:08 smithi145 bash[79878]: cluster 2024-03-07T03:38:07.413819+0000 mgr.y (mgr.34107) 788 : cluster 0 pgmap v1344: 105 pgs: 105 active+clean; 100 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.9 MiB/s wr, 38 op/s 2024-03-07T03:38:09.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:08 smithi145 bash[82329]: cluster 2024-03-07T03:38:07.413819+0000 mgr.y (mgr.34107) 788 : cluster 0 pgmap v1344: 105 pgs: 105 active+clean; 100 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.9 MiB/s wr, 38 op/s 2024-03-07T03:38:09.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:08 smithi189 bash[66969]: cluster 2024-03-07T03:38:07.413819+0000 mgr.y (mgr.34107) 788 : cluster 0 pgmap v1344: 105 pgs: 105 active+clean; 100 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.9 MiB/s wr, 38 op/s 2024-03-07T03:38:11.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:10 smithi145 bash[82329]: cluster 2024-03-07T03:38:09.414872+0000 mgr.y (mgr.34107) 789 : cluster 0 pgmap v1345: 105 pgs: 105 active+clean; 104 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 3.3 MiB/s wr, 38 op/s 2024-03-07T03:38:11.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:10 smithi145 bash[79878]: cluster 2024-03-07T03:38:09.414872+0000 mgr.y (mgr.34107) 789 : cluster 0 pgmap v1345: 105 pgs: 105 active+clean; 104 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 3.3 MiB/s wr, 38 op/s 2024-03-07T03:38:11.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:10 smithi189 bash[66969]: cluster 2024-03-07T03:38:09.414872+0000 mgr.y (mgr.34107) 789 : cluster 0 pgmap v1345: 105 pgs: 105 active+clean; 104 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 3.3 MiB/s wr, 38 op/s 2024-03-07T03:38:12.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:38:12 smithi189 bash[59814]: ts=2024-03-07T03:38:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:38:13.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:12 smithi145 bash[79878]: cluster 2024-03-07T03:38:11.415780+0000 mgr.y (mgr.34107) 790 : cluster 0 pgmap v1346: 105 pgs: 105 active+clean; 112 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 78 KiB/s rd, 3.5 MiB/s wr, 38 op/s 2024-03-07T03:38:13.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:38:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:38:12] "GET /metrics HTTP/1.1" 200 36568 "" "Prometheus/2.43.0" 2024-03-07T03:38:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:12 smithi145 bash[82329]: cluster 2024-03-07T03:38:11.415780+0000 mgr.y (mgr.34107) 790 : cluster 0 pgmap v1346: 105 pgs: 105 active+clean; 112 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 78 KiB/s rd, 3.5 MiB/s wr, 38 op/s 2024-03-07T03:38:13.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:12 smithi189 bash[66969]: cluster 2024-03-07T03:38:11.415780+0000 mgr.y (mgr.34107) 790 : cluster 0 pgmap v1346: 105 pgs: 105 active+clean; 112 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 78 KiB/s rd, 3.5 MiB/s wr, 38 op/s 2024-03-07T03:38:13.615 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/4.Grow (18527 ms) 2024-03-07T03:38:13.615 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/4.Shrink 2024-03-07T03:38:13.615 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:38:15.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:14 smithi145 bash[79878]: cluster 2024-03-07T03:38:13.416413+0000 mgr.y (mgr.34107) 791 : cluster 0 pgmap v1347: 105 pgs: 105 active+clean; 120 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 17 op/s 2024-03-07T03:38:15.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:14 smithi145 bash[82329]: cluster 2024-03-07T03:38:13.416413+0000 mgr.y (mgr.34107) 791 : cluster 0 pgmap v1347: 105 pgs: 105 active+clean; 120 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 17 op/s 2024-03-07T03:38:15.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:14 smithi189 bash[66969]: cluster 2024-03-07T03:38:13.416413+0000 mgr.y (mgr.34107) 791 : cluster 0 pgmap v1347: 105 pgs: 105 active+clean; 120 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.8 MiB/s wr, 17 op/s 2024-03-07T03:38:17.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:16 smithi145 bash[79878]: cluster 2024-03-07T03:38:15.417361+0000 mgr.y (mgr.34107) 792 : cluster 0 pgmap v1348: 105 pgs: 105 active+clean; 125 MiB data, 925 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.8 MiB/s wr, 17 op/s 2024-03-07T03:38:17.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:16 smithi145 bash[82329]: cluster 2024-03-07T03:38:15.417361+0000 mgr.y (mgr.34107) 792 : cluster 0 pgmap v1348: 105 pgs: 105 active+clean; 125 MiB data, 925 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.8 MiB/s wr, 17 op/s 2024-03-07T03:38:17.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:16 smithi189 bash[66969]: cluster 2024-03-07T03:38:15.417361+0000 mgr.y (mgr.34107) 792 : cluster 0 pgmap v1348: 105 pgs: 105 active+clean; 125 MiB data, 925 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.8 MiB/s wr, 17 op/s 2024-03-07T03:38:18.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:17 smithi145 bash[79878]: audit 2024-03-07T03:38:17.666705+0000 mon.a (mon.0) 1631 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:18.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:17 smithi145 bash[82329]: audit 2024-03-07T03:38:17.666705+0000 mon.a (mon.0) 1631 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:18.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:17 smithi189 bash[66969]: audit 2024-03-07T03:38:17.666705+0000 mon.a (mon.0) 1631 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:19.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:18 smithi145 bash[82329]: cluster 2024-03-07T03:38:17.418340+0000 mgr.y (mgr.34107) 793 : cluster 0 pgmap v1349: 105 pgs: 105 active+clean; 141 MiB data, 967 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 30 op/s 2024-03-07T03:38:19.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:18 smithi145 bash[82329]: cluster 2024-03-07T03:38:17.693446+0000 mon.a (mon.0) 1632 : cluster 0 osdmap e932: 8 total, 8 up, 8 in 2024-03-07T03:38:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:18 smithi145 bash[79878]: cluster 2024-03-07T03:38:17.418340+0000 mgr.y (mgr.34107) 793 : cluster 0 pgmap v1349: 105 pgs: 105 active+clean; 141 MiB data, 967 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 30 op/s 2024-03-07T03:38:19.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:18 smithi145 bash[79878]: cluster 2024-03-07T03:38:17.693446+0000 mon.a (mon.0) 1632 : cluster 0 osdmap e932: 8 total, 8 up, 8 in 2024-03-07T03:38:19.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:18 smithi189 bash[66969]: cluster 2024-03-07T03:38:17.418340+0000 mgr.y (mgr.34107) 793 : cluster 0 pgmap v1349: 105 pgs: 105 active+clean; 141 MiB data, 967 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 30 op/s 2024-03-07T03:38:19.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:18 smithi189 bash[66969]: cluster 2024-03-07T03:38:17.693446+0000 mon.a (mon.0) 1632 : cluster 0 osdmap e932: 8 total, 8 up, 8 in 2024-03-07T03:38:21.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:20 smithi145 bash[79878]: cluster 2024-03-07T03:38:19.419115+0000 mgr.y (mgr.34107) 794 : cluster 0 pgmap v1351: 105 pgs: 105 active+clean; 149 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.1 MiB/s wr, 62 op/s 2024-03-07T03:38:21.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:20 smithi145 bash[79878]: audit 2024-03-07T03:38:20.111985+0000 mon.a (mon.0) 1633 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:38:21.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:20 smithi145 bash[82329]: cluster 2024-03-07T03:38:19.419115+0000 mgr.y (mgr.34107) 794 : cluster 0 pgmap v1351: 105 pgs: 105 active+clean; 149 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.1 MiB/s wr, 62 op/s 2024-03-07T03:38:21.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:20 smithi145 bash[82329]: audit 2024-03-07T03:38:20.111985+0000 mon.a (mon.0) 1633 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:38:21.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:20 smithi189 bash[66969]: cluster 2024-03-07T03:38:19.419115+0000 mgr.y (mgr.34107) 794 : cluster 0 pgmap v1351: 105 pgs: 105 active+clean; 149 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.1 MiB/s wr, 62 op/s 2024-03-07T03:38:21.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:20 smithi189 bash[66969]: audit 2024-03-07T03:38:20.111985+0000 mon.a (mon.0) 1633 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:38:22.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:21 smithi145 bash[79878]: audit 2024-03-07T03:38:20.736479+0000 mon.a (mon.0) 1634 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:22.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:21 smithi145 bash[79878]: audit 2024-03-07T03:38:20.744507+0000 mon.a (mon.0) 1635 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:22.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:21 smithi145 bash[82329]: audit 2024-03-07T03:38:20.736479+0000 mon.a (mon.0) 1634 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:22.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:21 smithi145 bash[82329]: audit 2024-03-07T03:38:20.744507+0000 mon.a (mon.0) 1635 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:22.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:21 smithi189 bash[66969]: audit 2024-03-07T03:38:20.736479+0000 mon.a (mon.0) 1634 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:22.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:21 smithi189 bash[66969]: audit 2024-03-07T03:38:20.744507+0000 mon.a (mon.0) 1635 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:22.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:38:22 smithi189 bash[59814]: ts=2024-03-07T03:38:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:38:23.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:22 smithi145 bash[79878]: cluster 2024-03-07T03:38:21.419798+0000 mgr.y (mgr.34107) 795 : cluster 0 pgmap v1352: 105 pgs: 105 active+clean; 165 MiB data, 1017 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.9 MiB/s wr, 64 op/s 2024-03-07T03:38:23.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:38:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:38:22] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.43.0" 2024-03-07T03:38:23.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:22 smithi145 bash[82329]: cluster 2024-03-07T03:38:21.419798+0000 mgr.y (mgr.34107) 795 : cluster 0 pgmap v1352: 105 pgs: 105 active+clean; 165 MiB data, 1017 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.9 MiB/s wr, 64 op/s 2024-03-07T03:38:23.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:22 smithi189 bash[66969]: cluster 2024-03-07T03:38:21.419798+0000 mgr.y (mgr.34107) 795 : cluster 0 pgmap v1352: 105 pgs: 105 active+clean; 165 MiB data, 1017 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.9 MiB/s wr, 64 op/s 2024-03-07T03:38:25.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:24 smithi145 bash[79878]: cluster 2024-03-07T03:38:23.420379+0000 mgr.y (mgr.34107) 796 : cluster 0 pgmap v1353: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.6 MiB/s wr, 59 op/s 2024-03-07T03:38:25.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:24 smithi145 bash[82329]: cluster 2024-03-07T03:38:23.420379+0000 mgr.y (mgr.34107) 796 : cluster 0 pgmap v1353: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.6 MiB/s wr, 59 op/s 2024-03-07T03:38:25.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:24 smithi189 bash[66969]: cluster 2024-03-07T03:38:23.420379+0000 mgr.y (mgr.34107) 796 : cluster 0 pgmap v1353: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.6 MiB/s wr, 59 op/s 2024-03-07T03:38:27.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:26 smithi145 bash[79878]: cluster 2024-03-07T03:38:25.421387+0000 mgr.y (mgr.34107) 797 : cluster 0 pgmap v1354: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 5.7 MiB/s wr, 63 op/s 2024-03-07T03:38:27.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:26 smithi145 bash[79878]: audit 2024-03-07T03:38:26.146397+0000 mon.a (mon.0) 1636 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:27.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:26 smithi145 bash[79878]: audit 2024-03-07T03:38:26.152065+0000 mon.a (mon.0) 1637 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:27.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:26 smithi145 bash[79878]: audit 2024-03-07T03:38:26.529253+0000 mon.a (mon.0) 1638 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:27.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:26 smithi145 bash[79878]: audit 2024-03-07T03:38:26.535135+0000 mon.a (mon.0) 1639 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:27.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:26 smithi145 bash[82329]: cluster 2024-03-07T03:38:25.421387+0000 mgr.y (mgr.34107) 797 : cluster 0 pgmap v1354: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 5.7 MiB/s wr, 63 op/s 2024-03-07T03:38:27.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:26 smithi145 bash[82329]: audit 2024-03-07T03:38:26.146397+0000 mon.a (mon.0) 1636 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:27.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:26 smithi145 bash[82329]: audit 2024-03-07T03:38:26.152065+0000 mon.a (mon.0) 1637 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:27.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:26 smithi145 bash[82329]: audit 2024-03-07T03:38:26.529253+0000 mon.a (mon.0) 1638 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:27.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:26 smithi145 bash[82329]: audit 2024-03-07T03:38:26.535135+0000 mon.a (mon.0) 1639 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:27.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:26 smithi189 bash[66969]: cluster 2024-03-07T03:38:25.421387+0000 mgr.y (mgr.34107) 797 : cluster 0 pgmap v1354: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 5.7 MiB/s wr, 63 op/s 2024-03-07T03:38:27.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:26 smithi189 bash[66969]: audit 2024-03-07T03:38:26.146397+0000 mon.a (mon.0) 1636 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:27.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:26 smithi189 bash[66969]: audit 2024-03-07T03:38:26.152065+0000 mon.a (mon.0) 1637 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:27.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:26 smithi189 bash[66969]: audit 2024-03-07T03:38:26.529253+0000 mon.a (mon.0) 1638 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:27.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:26 smithi189 bash[66969]: audit 2024-03-07T03:38:26.535135+0000 mon.a (mon.0) 1639 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:28.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:27 smithi145 bash[79878]: audit 2024-03-07T03:38:27.116620+0000 mon.a (mon.0) 1640 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:38:28.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:27 smithi145 bash[79878]: audit 2024-03-07T03:38:27.118453+0000 mon.a (mon.0) 1641 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:38:28.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:27 smithi145 bash[79878]: audit 2024-03-07T03:38:27.124809+0000 mon.a (mon.0) 1642 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:28.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:27 smithi145 bash[82329]: audit 2024-03-07T03:38:27.116620+0000 mon.a (mon.0) 1640 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:38:28.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:27 smithi145 bash[82329]: audit 2024-03-07T03:38:27.118453+0000 mon.a (mon.0) 1641 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:38:28.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:27 smithi145 bash[82329]: audit 2024-03-07T03:38:27.124809+0000 mon.a (mon.0) 1642 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:28.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:27 smithi189 bash[66969]: audit 2024-03-07T03:38:27.116620+0000 mon.a (mon.0) 1640 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:38:28.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:27 smithi189 bash[66969]: audit 2024-03-07T03:38:27.118453+0000 mon.a (mon.0) 1641 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:38:28.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:27 smithi189 bash[66969]: audit 2024-03-07T03:38:27.124809+0000 mon.a (mon.0) 1642 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:38:28.919 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/4.Shrink (15304 ms) 2024-03-07T03:38:28.919 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/4.ShrinkToOne 2024-03-07T03:38:28.919 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:38:29.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:28 smithi145 bash[79878]: cluster 2024-03-07T03:38:27.422106+0000 mgr.y (mgr.34107) 798 : cluster 0 pgmap v1355: 105 pgs: 105 active+clean; 186 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 4.5 MiB/s wr, 46 op/s 2024-03-07T03:38:29.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:28 smithi145 bash[82329]: cluster 2024-03-07T03:38:27.422106+0000 mgr.y (mgr.34107) 798 : cluster 0 pgmap v1355: 105 pgs: 105 active+clean; 186 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 4.5 MiB/s wr, 46 op/s 2024-03-07T03:38:29.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:28 smithi189 bash[66969]: cluster 2024-03-07T03:38:27.422106+0000 mgr.y (mgr.34107) 798 : cluster 0 pgmap v1355: 105 pgs: 105 active+clean; 186 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 4.5 MiB/s wr, 46 op/s 2024-03-07T03:38:31.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:30 smithi145 bash[82329]: cluster 2024-03-07T03:38:29.422965+0000 mgr.y (mgr.34107) 799 : cluster 0 pgmap v1356: 105 pgs: 105 active+clean; 190 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.2 MiB/s wr, 40 op/s 2024-03-07T03:38:31.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:30 smithi145 bash[79878]: cluster 2024-03-07T03:38:29.422965+0000 mgr.y (mgr.34107) 799 : cluster 0 pgmap v1356: 105 pgs: 105 active+clean; 190 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.2 MiB/s wr, 40 op/s 2024-03-07T03:38:31.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:30 smithi189 bash[66969]: cluster 2024-03-07T03:38:29.422965+0000 mgr.y (mgr.34107) 799 : cluster 0 pgmap v1356: 105 pgs: 105 active+clean; 190 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.2 MiB/s wr, 40 op/s 2024-03-07T03:38:32.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:38:32 smithi189 bash[59814]: ts=2024-03-07T03:38:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:38:33.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:38:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:38:32] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.43.0" 2024-03-07T03:38:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:32 smithi145 bash[79878]: cluster 2024-03-07T03:38:31.423873+0000 mgr.y (mgr.34107) 800 : cluster 0 pgmap v1357: 105 pgs: 105 active+clean; 206 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 5.4 MiB/s wr, 32 op/s 2024-03-07T03:38:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:32 smithi145 bash[79878]: audit 2024-03-07T03:38:32.667005+0000 mon.a (mon.0) 1643 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:32 smithi145 bash[82329]: cluster 2024-03-07T03:38:31.423873+0000 mgr.y (mgr.34107) 800 : cluster 0 pgmap v1357: 105 pgs: 105 active+clean; 206 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 5.4 MiB/s wr, 32 op/s 2024-03-07T03:38:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:32 smithi145 bash[82329]: audit 2024-03-07T03:38:32.667005+0000 mon.a (mon.0) 1643 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:33.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:32 smithi189 bash[66969]: cluster 2024-03-07T03:38:31.423873+0000 mgr.y (mgr.34107) 800 : cluster 0 pgmap v1357: 105 pgs: 105 active+clean; 206 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 5.4 MiB/s wr, 32 op/s 2024-03-07T03:38:33.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:32 smithi189 bash[66969]: audit 2024-03-07T03:38:32.667005+0000 mon.a (mon.0) 1643 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:35.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:34 smithi145 bash[79878]: cluster 2024-03-07T03:38:33.424762+0000 mgr.y (mgr.34107) 801 : cluster 0 pgmap v1358: 105 pgs: 105 active+clean; 211 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.2 MiB/s wr, 27 op/s 2024-03-07T03:38:35.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:34 smithi145 bash[79878]: cluster 2024-03-07T03:38:33.776633+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e933: 8 total, 8 up, 8 in 2024-03-07T03:38:35.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:34 smithi145 bash[82329]: cluster 2024-03-07T03:38:33.424762+0000 mgr.y (mgr.34107) 801 : cluster 0 pgmap v1358: 105 pgs: 105 active+clean; 211 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.2 MiB/s wr, 27 op/s 2024-03-07T03:38:35.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:34 smithi145 bash[82329]: cluster 2024-03-07T03:38:33.776633+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e933: 8 total, 8 up, 8 in 2024-03-07T03:38:35.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:34 smithi189 bash[66969]: cluster 2024-03-07T03:38:33.424762+0000 mgr.y (mgr.34107) 801 : cluster 0 pgmap v1358: 105 pgs: 105 active+clean; 211 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.2 MiB/s wr, 27 op/s 2024-03-07T03:38:35.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:34 smithi189 bash[66969]: cluster 2024-03-07T03:38:33.776633+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e933: 8 total, 8 up, 8 in 2024-03-07T03:38:37.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:36 smithi145 bash[79878]: cluster 2024-03-07T03:38:35.425690+0000 mgr.y (mgr.34107) 802 : cluster 0 pgmap v1360: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.6 MiB/s wr, 53 op/s 2024-03-07T03:38:37.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:36 smithi145 bash[82329]: cluster 2024-03-07T03:38:35.425690+0000 mgr.y (mgr.34107) 802 : cluster 0 pgmap v1360: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.6 MiB/s wr, 53 op/s 2024-03-07T03:38:37.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:36 smithi189 bash[66969]: cluster 2024-03-07T03:38:35.425690+0000 mgr.y (mgr.34107) 802 : cluster 0 pgmap v1360: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.6 MiB/s wr, 53 op/s 2024-03-07T03:38:39.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:38 smithi145 bash[82329]: cluster 2024-03-07T03:38:37.426412+0000 mgr.y (mgr.34107) 803 : cluster 0 pgmap v1361: 105 pgs: 105 active+clean; 240 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.0 MiB/s wr, 60 op/s 2024-03-07T03:38:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:38 smithi145 bash[79878]: cluster 2024-03-07T03:38:37.426412+0000 mgr.y (mgr.34107) 803 : cluster 0 pgmap v1361: 105 pgs: 105 active+clean; 240 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.0 MiB/s wr, 60 op/s 2024-03-07T03:38:39.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:38 smithi189 bash[66969]: cluster 2024-03-07T03:38:37.426412+0000 mgr.y (mgr.34107) 803 : cluster 0 pgmap v1361: 105 pgs: 105 active+clean; 240 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.0 MiB/s wr, 60 op/s 2024-03-07T03:38:41.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:40 smithi145 bash[79878]: cluster 2024-03-07T03:38:39.427352+0000 mgr.y (mgr.34107) 804 : cluster 0 pgmap v1362: 105 pgs: 105 active+clean; 240 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.6 MiB/s wr, 59 op/s 2024-03-07T03:38:41.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:40 smithi145 bash[82329]: cluster 2024-03-07T03:38:39.427352+0000 mgr.y (mgr.34107) 804 : cluster 0 pgmap v1362: 105 pgs: 105 active+clean; 240 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.6 MiB/s wr, 59 op/s 2024-03-07T03:38:41.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:40 smithi189 bash[66969]: cluster 2024-03-07T03:38:39.427352+0000 mgr.y (mgr.34107) 804 : cluster 0 pgmap v1362: 105 pgs: 105 active+clean; 240 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.6 MiB/s wr, 59 op/s 2024-03-07T03:38:42.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:38:42 smithi189 bash[59814]: ts=2024-03-07T03:38:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:38:43.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:38:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:38:42] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.43.0" 2024-03-07T03:38:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:42 smithi145 bash[79878]: cluster 2024-03-07T03:38:41.428040+0000 mgr.y (mgr.34107) 805 : cluster 0 pgmap v1363: 105 pgs: 105 active+clean; 240 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.2 MiB/s wr, 41 op/s 2024-03-07T03:38:43.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:42 smithi145 bash[82329]: cluster 2024-03-07T03:38:41.428040+0000 mgr.y (mgr.34107) 805 : cluster 0 pgmap v1363: 105 pgs: 105 active+clean; 240 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.2 MiB/s wr, 41 op/s 2024-03-07T03:38:43.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:42 smithi189 bash[66969]: cluster 2024-03-07T03:38:41.428040+0000 mgr.y (mgr.34107) 805 : cluster 0 pgmap v1363: 105 pgs: 105 active+clean; 240 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.2 MiB/s wr, 41 op/s 2024-03-07T03:38:45.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:44 smithi145 bash[79878]: cluster 2024-03-07T03:38:43.428551+0000 mgr.y (mgr.34107) 806 : cluster 0 pgmap v1364: 105 pgs: 105 active+clean; 244 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-03-07T03:38:45.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:44 smithi145 bash[82329]: cluster 2024-03-07T03:38:43.428551+0000 mgr.y (mgr.34107) 806 : cluster 0 pgmap v1364: 105 pgs: 105 active+clean; 244 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-03-07T03:38:45.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:44 smithi189 bash[66969]: cluster 2024-03-07T03:38:43.428551+0000 mgr.y (mgr.34107) 806 : cluster 0 pgmap v1364: 105 pgs: 105 active+clean; 244 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-03-07T03:38:47.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:46 smithi145 bash[79878]: cluster 2024-03-07T03:38:45.429209+0000 mgr.y (mgr.34107) 807 : cluster 0 pgmap v1365: 105 pgs: 105 active+clean; 248 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 2.7 MiB/s wr, 33 op/s 2024-03-07T03:38:47.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:46 smithi145 bash[82329]: cluster 2024-03-07T03:38:45.429209+0000 mgr.y (mgr.34107) 807 : cluster 0 pgmap v1365: 105 pgs: 105 active+clean; 248 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 2.7 MiB/s wr, 33 op/s 2024-03-07T03:38:47.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:46 smithi189 bash[66969]: cluster 2024-03-07T03:38:45.429209+0000 mgr.y (mgr.34107) 807 : cluster 0 pgmap v1365: 105 pgs: 105 active+clean; 248 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 2.7 MiB/s wr, 33 op/s 2024-03-07T03:38:47.133 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/4.ShrinkToOne (18214 ms) 2024-03-07T03:38:47.133 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/4.ShrinkToOneAfterSnapshot 2024-03-07T03:38:47.133 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:38:48.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:47 smithi145 bash[79878]: audit 2024-03-07T03:38:47.667366+0000 mon.a (mon.0) 1645 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:48.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:47 smithi145 bash[82329]: audit 2024-03-07T03:38:47.667366+0000 mon.a (mon.0) 1645 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:48.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:47 smithi189 bash[66969]: audit 2024-03-07T03:38:47.667366+0000 mon.a (mon.0) 1645 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:49.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:48 smithi145 bash[79878]: cluster 2024-03-07T03:38:47.430197+0000 mgr.y (mgr.34107) 808 : cluster 0 pgmap v1366: 105 pgs: 105 active+clean; 256 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.0 MiB/s wr, 24 op/s 2024-03-07T03:38:49.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:48 smithi145 bash[82329]: cluster 2024-03-07T03:38:47.430197+0000 mgr.y (mgr.34107) 808 : cluster 0 pgmap v1366: 105 pgs: 105 active+clean; 256 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.0 MiB/s wr, 24 op/s 2024-03-07T03:38:49.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:48 smithi189 bash[66969]: cluster 2024-03-07T03:38:47.430197+0000 mgr.y (mgr.34107) 808 : cluster 0 pgmap v1366: 105 pgs: 105 active+clean; 256 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.0 MiB/s wr, 24 op/s 2024-03-07T03:38:51.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:50 smithi145 bash[79878]: cluster 2024-03-07T03:38:49.431077+0000 mgr.y (mgr.34107) 809 : cluster 0 pgmap v1367: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-03-07T03:38:51.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:50 smithi145 bash[82329]: cluster 2024-03-07T03:38:49.431077+0000 mgr.y (mgr.34107) 809 : cluster 0 pgmap v1367: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-03-07T03:38:51.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:50 smithi189 bash[66969]: cluster 2024-03-07T03:38:49.431077+0000 mgr.y (mgr.34107) 809 : cluster 0 pgmap v1367: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-03-07T03:38:52.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:38:52 smithi189 bash[59814]: ts=2024-03-07T03:38:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:38:53.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:38:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:38:52] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.43.0" 2024-03-07T03:38:53.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:52 smithi145 bash[79878]: cluster 2024-03-07T03:38:51.432072+0000 mgr.y (mgr.34107) 810 : cluster 0 pgmap v1368: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 20 op/s 2024-03-07T03:38:53.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:52 smithi145 bash[79878]: cluster 2024-03-07T03:38:51.824033+0000 mon.a (mon.0) 1646 : cluster 0 osdmap e934: 8 total, 8 up, 8 in 2024-03-07T03:38:53.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:52 smithi145 bash[82329]: cluster 2024-03-07T03:38:51.432072+0000 mgr.y (mgr.34107) 810 : cluster 0 pgmap v1368: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 20 op/s 2024-03-07T03:38:53.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:52 smithi145 bash[82329]: cluster 2024-03-07T03:38:51.824033+0000 mon.a (mon.0) 1646 : cluster 0 osdmap e934: 8 total, 8 up, 8 in 2024-03-07T03:38:53.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:52 smithi189 bash[66969]: cluster 2024-03-07T03:38:51.432072+0000 mgr.y (mgr.34107) 810 : cluster 0 pgmap v1368: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 20 op/s 2024-03-07T03:38:53.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:52 smithi189 bash[66969]: cluster 2024-03-07T03:38:51.824033+0000 mon.a (mon.0) 1646 : cluster 0 osdmap e934: 8 total, 8 up, 8 in 2024-03-07T03:38:55.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:54 smithi189 bash[66969]: cluster 2024-03-07T03:38:53.432521+0000 mgr.y (mgr.34107) 811 : cluster 0 pgmap v1370: 105 pgs: 105 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.5 MiB/s wr, 28 op/s 2024-03-07T03:38:55.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:54 smithi145 bash[79878]: cluster 2024-03-07T03:38:53.432521+0000 mgr.y (mgr.34107) 811 : cluster 0 pgmap v1370: 105 pgs: 105 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.5 MiB/s wr, 28 op/s 2024-03-07T03:38:55.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:54 smithi145 bash[82329]: cluster 2024-03-07T03:38:53.432521+0000 mgr.y (mgr.34107) 811 : cluster 0 pgmap v1370: 105 pgs: 105 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.5 MiB/s wr, 28 op/s 2024-03-07T03:38:57.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:56 smithi189 bash[66969]: cluster 2024-03-07T03:38:55.433275+0000 mgr.y (mgr.34107) 812 : cluster 0 pgmap v1371: 105 pgs: 105 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.3 MiB/s wr, 34 op/s 2024-03-07T03:38:57.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:56 smithi145 bash[79878]: cluster 2024-03-07T03:38:55.433275+0000 mgr.y (mgr.34107) 812 : cluster 0 pgmap v1371: 105 pgs: 105 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.3 MiB/s wr, 34 op/s 2024-03-07T03:38:57.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:56 smithi145 bash[82329]: cluster 2024-03-07T03:38:55.433275+0000 mgr.y (mgr.34107) 812 : cluster 0 pgmap v1371: 105 pgs: 105 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.3 MiB/s wr, 34 op/s 2024-03-07T03:38:59.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:38:58 smithi189 bash[66969]: cluster 2024-03-07T03:38:57.433977+0000 mgr.y (mgr.34107) 813 : cluster 0 pgmap v1372: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 441 KiB/s rd, 4.1 MiB/s wr, 41 op/s 2024-03-07T03:38:59.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:38:58 smithi145 bash[79878]: cluster 2024-03-07T03:38:57.433977+0000 mgr.y (mgr.34107) 813 : cluster 0 pgmap v1372: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 441 KiB/s rd, 4.1 MiB/s wr, 41 op/s 2024-03-07T03:38:59.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:38:58 smithi145 bash[82329]: cluster 2024-03-07T03:38:57.433977+0000 mgr.y (mgr.34107) 813 : cluster 0 pgmap v1372: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 441 KiB/s rd, 4.1 MiB/s wr, 41 op/s 2024-03-07T03:39:01.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:00 smithi189 bash[66969]: cluster 2024-03-07T03:38:59.434622+0000 mgr.y (mgr.34107) 814 : cluster 0 pgmap v1373: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 3.3 MiB/s wr, 39 op/s 2024-03-07T03:39:01.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:00 smithi145 bash[79878]: cluster 2024-03-07T03:38:59.434622+0000 mgr.y (mgr.34107) 814 : cluster 0 pgmap v1373: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 3.3 MiB/s wr, 39 op/s 2024-03-07T03:39:01.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:00 smithi145 bash[82329]: cluster 2024-03-07T03:38:59.434622+0000 mgr.y (mgr.34107) 814 : cluster 0 pgmap v1373: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 3.3 MiB/s wr, 39 op/s 2024-03-07T03:39:02.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:39:02 smithi189 bash[59814]: ts=2024-03-07T03:39:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:39:03.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:39:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:39:02] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.43.0" 2024-03-07T03:39:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:02 smithi145 bash[79878]: cluster 2024-03-07T03:39:01.435174+0000 mgr.y (mgr.34107) 815 : cluster 0 pgmap v1374: 105 pgs: 105 active+clean; 297 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 3.7 MiB/s wr, 41 op/s 2024-03-07T03:39:03.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:02 smithi145 bash[79878]: audit 2024-03-07T03:39:02.667812+0000 mon.a (mon.0) 1647 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:02 smithi145 bash[82329]: cluster 2024-03-07T03:39:01.435174+0000 mgr.y (mgr.34107) 815 : cluster 0 pgmap v1374: 105 pgs: 105 active+clean; 297 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 3.7 MiB/s wr, 41 op/s 2024-03-07T03:39:03.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:02 smithi145 bash[82329]: audit 2024-03-07T03:39:02.667812+0000 mon.a (mon.0) 1647 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:03.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:02 smithi189 bash[66969]: cluster 2024-03-07T03:39:01.435174+0000 mgr.y (mgr.34107) 815 : cluster 0 pgmap v1374: 105 pgs: 105 active+clean; 297 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 3.7 MiB/s wr, 41 op/s 2024-03-07T03:39:03.078 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:02 smithi189 bash[66969]: audit 2024-03-07T03:39:02.667812+0000 mon.a (mon.0) 1647 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:05.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:04 smithi145 bash[79878]: cluster 2024-03-07T03:39:03.435950+0000 mgr.y (mgr.34107) 816 : cluster 0 pgmap v1375: 105 pgs: 105 active+clean; 305 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 3.1 MiB/s wr, 36 op/s 2024-03-07T03:39:05.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:04 smithi145 bash[79878]: cluster 2024-03-07T03:39:03.847756+0000 mon.a (mon.0) 1648 : cluster 0 osdmap e935: 8 total, 8 up, 8 in 2024-03-07T03:39:05.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:04 smithi145 bash[82329]: cluster 2024-03-07T03:39:03.435950+0000 mgr.y (mgr.34107) 816 : cluster 0 pgmap v1375: 105 pgs: 105 active+clean; 305 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 3.1 MiB/s wr, 36 op/s 2024-03-07T03:39:05.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:04 smithi145 bash[82329]: cluster 2024-03-07T03:39:03.847756+0000 mon.a (mon.0) 1648 : cluster 0 osdmap e935: 8 total, 8 up, 8 in 2024-03-07T03:39:05.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:04 smithi189 bash[66969]: cluster 2024-03-07T03:39:03.435950+0000 mgr.y (mgr.34107) 816 : cluster 0 pgmap v1375: 105 pgs: 105 active+clean; 305 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 3.1 MiB/s wr, 36 op/s 2024-03-07T03:39:05.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:04 smithi189 bash[66969]: cluster 2024-03-07T03:39:03.847756+0000 mon.a (mon.0) 1648 : cluster 0 osdmap e935: 8 total, 8 up, 8 in 2024-03-07T03:39:06.045 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/4.ShrinkToOneAfterSnapshot (18912 ms) 2024-03-07T03:39:06.045 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/4.MinOverlap 2024-03-07T03:39:06.045 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:39:07.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:06 smithi145 bash[82329]: cluster 2024-03-07T03:39:05.436539+0000 mgr.y (mgr.34107) 817 : cluster 0 pgmap v1377: 105 pgs: 105 active+clean; 321 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.0 MiB/s wr, 34 op/s 2024-03-07T03:39:07.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:06 smithi145 bash[79878]: cluster 2024-03-07T03:39:05.436539+0000 mgr.y (mgr.34107) 817 : cluster 0 pgmap v1377: 105 pgs: 105 active+clean; 321 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.0 MiB/s wr, 34 op/s 2024-03-07T03:39:07.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:06 smithi189 bash[66969]: cluster 2024-03-07T03:39:05.436539+0000 mgr.y (mgr.34107) 817 : cluster 0 pgmap v1377: 105 pgs: 105 active+clean; 321 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.0 MiB/s wr, 34 op/s 2024-03-07T03:39:09.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:08 smithi145 bash[82329]: cluster 2024-03-07T03:39:07.437467+0000 mgr.y (mgr.34107) 818 : cluster 0 pgmap v1378: 105 pgs: 105 active+clean; 366 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 7.7 MiB/s wr, 44 op/s 2024-03-07T03:39:09.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:08 smithi145 bash[79878]: cluster 2024-03-07T03:39:07.437467+0000 mgr.y (mgr.34107) 818 : cluster 0 pgmap v1378: 105 pgs: 105 active+clean; 366 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 7.7 MiB/s wr, 44 op/s 2024-03-07T03:39:09.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:08 smithi189 bash[66969]: cluster 2024-03-07T03:39:07.437467+0000 mgr.y (mgr.34107) 818 : cluster 0 pgmap v1378: 105 pgs: 105 active+clean; 366 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 7.7 MiB/s wr, 44 op/s 2024-03-07T03:39:11.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:10 smithi145 bash[79878]: cluster 2024-03-07T03:39:09.438369+0000 mgr.y (mgr.34107) 819 : cluster 0 pgmap v1379: 105 pgs: 105 active+clean; 374 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.5 MiB/s wr, 47 op/s 2024-03-07T03:39:11.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:10 smithi145 bash[82329]: cluster 2024-03-07T03:39:09.438369+0000 mgr.y (mgr.34107) 819 : cluster 0 pgmap v1379: 105 pgs: 105 active+clean; 374 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.5 MiB/s wr, 47 op/s 2024-03-07T03:39:11.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:10 smithi189 bash[66969]: cluster 2024-03-07T03:39:09.438369+0000 mgr.y (mgr.34107) 819 : cluster 0 pgmap v1379: 105 pgs: 105 active+clean; 374 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.5 MiB/s wr, 47 op/s 2024-03-07T03:39:12.302 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:11 smithi189 bash[66969]: cluster 2024-03-07T03:39:10.876464+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e936: 8 total, 8 up, 8 in 2024-03-07T03:39:12.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:11 smithi145 bash[79878]: cluster 2024-03-07T03:39:10.876464+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e936: 8 total, 8 up, 8 in 2024-03-07T03:39:12.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:11 smithi145 bash[82329]: cluster 2024-03-07T03:39:10.876464+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e936: 8 total, 8 up, 8 in 2024-03-07T03:39:12.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:39:12 smithi189 bash[59814]: ts=2024-03-07T03:39:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:39:13.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:12 smithi145 bash[79878]: cluster 2024-03-07T03:39:11.439001+0000 mgr.y (mgr.34107) 820 : cluster 0 pgmap v1381: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 11 MiB/s wr, 58 op/s 2024-03-07T03:39:13.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:39:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:39:12] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.43.0" 2024-03-07T03:39:13.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:12 smithi145 bash[82329]: cluster 2024-03-07T03:39:11.439001+0000 mgr.y (mgr.34107) 820 : cluster 0 pgmap v1381: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 11 MiB/s wr, 58 op/s 2024-03-07T03:39:13.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:12 smithi189 bash[66969]: cluster 2024-03-07T03:39:11.439001+0000 mgr.y (mgr.34107) 820 : cluster 0 pgmap v1381: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 11 MiB/s wr, 58 op/s 2024-03-07T03:39:15.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:14 smithi145 bash[79878]: cluster 2024-03-07T03:39:13.439737+0000 mgr.y (mgr.34107) 821 : cluster 0 pgmap v1382: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 9.0 MiB/s wr, 48 op/s 2024-03-07T03:39:15.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:14 smithi145 bash[82329]: cluster 2024-03-07T03:39:13.439737+0000 mgr.y (mgr.34107) 821 : cluster 0 pgmap v1382: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 9.0 MiB/s wr, 48 op/s 2024-03-07T03:39:15.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:14 smithi189 bash[66969]: cluster 2024-03-07T03:39:13.439737+0000 mgr.y (mgr.34107) 821 : cluster 0 pgmap v1382: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 9.0 MiB/s wr, 48 op/s 2024-03-07T03:39:17.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:16 smithi145 bash[79878]: cluster 2024-03-07T03:39:15.440444+0000 mgr.y (mgr.34107) 822 : cluster 0 pgmap v1383: 105 pgs: 105 active+clean; 403 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 8.2 MiB/s wr, 64 op/s 2024-03-07T03:39:17.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:16 smithi145 bash[82329]: cluster 2024-03-07T03:39:15.440444+0000 mgr.y (mgr.34107) 822 : cluster 0 pgmap v1383: 105 pgs: 105 active+clean; 403 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 8.2 MiB/s wr, 64 op/s 2024-03-07T03:39:17.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:16 smithi189 bash[66969]: cluster 2024-03-07T03:39:15.440444+0000 mgr.y (mgr.34107) 822 : cluster 0 pgmap v1383: 105 pgs: 105 active+clean; 403 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 8.2 MiB/s wr, 64 op/s 2024-03-07T03:39:18.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:17 smithi145 bash[79878]: audit 2024-03-07T03:39:17.667678+0000 mon.a (mon.0) 1650 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:18.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:17 smithi145 bash[82329]: audit 2024-03-07T03:39:17.667678+0000 mon.a (mon.0) 1650 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:18.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:17 smithi189 bash[66969]: audit 2024-03-07T03:39:17.667678+0000 mon.a (mon.0) 1650 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:19.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:18 smithi145 bash[79878]: cluster 2024-03-07T03:39:17.441117+0000 mgr.y (mgr.34107) 823 : cluster 0 pgmap v1384: 105 pgs: 105 active+clean; 403 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 3.7 MiB/s wr, 40 op/s 2024-03-07T03:39:19.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:18 smithi145 bash[82329]: cluster 2024-03-07T03:39:17.441117+0000 mgr.y (mgr.34107) 823 : cluster 0 pgmap v1384: 105 pgs: 105 active+clean; 403 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 3.7 MiB/s wr, 40 op/s 2024-03-07T03:39:19.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:18 smithi189 bash[66969]: cluster 2024-03-07T03:39:17.441117+0000 mgr.y (mgr.34107) 823 : cluster 0 pgmap v1384: 105 pgs: 105 active+clean; 403 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 3.7 MiB/s wr, 40 op/s 2024-03-07T03:39:21.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:20 smithi145 bash[79878]: cluster 2024-03-07T03:39:19.441758+0000 mgr.y (mgr.34107) 824 : cluster 0 pgmap v1385: 105 pgs: 105 active+clean; 403 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.9 MiB/s wr, 37 op/s 2024-03-07T03:39:21.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:20 smithi145 bash[82329]: cluster 2024-03-07T03:39:19.441758+0000 mgr.y (mgr.34107) 824 : cluster 0 pgmap v1385: 105 pgs: 105 active+clean; 403 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.9 MiB/s wr, 37 op/s 2024-03-07T03:39:21.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:20 smithi189 bash[66969]: cluster 2024-03-07T03:39:19.441758+0000 mgr.y (mgr.34107) 824 : cluster 0 pgmap v1385: 105 pgs: 105 active+clean; 403 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.9 MiB/s wr, 37 op/s 2024-03-07T03:39:22.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:39:22 smithi189 bash[59814]: ts=2024-03-07T03:39:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:39:23.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:22 smithi145 bash[79878]: cluster 2024-03-07T03:39:21.442328+0000 mgr.y (mgr.34107) 825 : cluster 0 pgmap v1386: 105 pgs: 105 active+clean; 415 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.3 MiB/s wr, 33 op/s 2024-03-07T03:39:23.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:39:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:39:22] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.43.0" 2024-03-07T03:39:23.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:22 smithi145 bash[82329]: cluster 2024-03-07T03:39:21.442328+0000 mgr.y (mgr.34107) 825 : cluster 0 pgmap v1386: 105 pgs: 105 active+clean; 415 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.3 MiB/s wr, 33 op/s 2024-03-07T03:39:23.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:22 smithi189 bash[66969]: cluster 2024-03-07T03:39:21.442328+0000 mgr.y (mgr.34107) 825 : cluster 0 pgmap v1386: 105 pgs: 105 active+clean; 415 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.3 MiB/s wr, 33 op/s 2024-03-07T03:39:25.251 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/4.MinOverlap (19206 ms) 2024-03-07T03:39:25.251 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/4.ZeroOverlap 2024-03-07T03:39:25.251 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:39:25.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:24 smithi145 bash[79878]: cluster 2024-03-07T03:39:23.442912+0000 mgr.y (mgr.34107) 826 : cluster 0 pgmap v1387: 105 pgs: 105 active+clean; 415 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 2.0 MiB/s wr, 29 op/s 2024-03-07T03:39:25.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:24 smithi145 bash[82329]: cluster 2024-03-07T03:39:23.442912+0000 mgr.y (mgr.34107) 826 : cluster 0 pgmap v1387: 105 pgs: 105 active+clean; 415 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 2.0 MiB/s wr, 29 op/s 2024-03-07T03:39:25.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:24 smithi189 bash[66969]: cluster 2024-03-07T03:39:23.442912+0000 mgr.y (mgr.34107) 826 : cluster 0 pgmap v1387: 105 pgs: 105 active+clean; 415 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 2.0 MiB/s wr, 29 op/s 2024-03-07T03:39:27.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:26 smithi145 bash[79878]: cluster 2024-03-07T03:39:25.443885+0000 mgr.y (mgr.34107) 827 : cluster 0 pgmap v1388: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 781 KiB/s rd, 3.0 MiB/s wr, 35 op/s 2024-03-07T03:39:27.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:26 smithi145 bash[82329]: cluster 2024-03-07T03:39:25.443885+0000 mgr.y (mgr.34107) 827 : cluster 0 pgmap v1388: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 781 KiB/s rd, 3.0 MiB/s wr, 35 op/s 2024-03-07T03:39:27.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:26 smithi189 bash[66969]: cluster 2024-03-07T03:39:25.443885+0000 mgr.y (mgr.34107) 827 : cluster 0 pgmap v1388: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 781 KiB/s rd, 3.0 MiB/s wr, 35 op/s 2024-03-07T03:39:28.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:27 smithi145 bash[79878]: audit 2024-03-07T03:39:27.212861+0000 mon.a (mon.0) 1651 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:39:28.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:27 smithi145 bash[79878]: audit 2024-03-07T03:39:27.740988+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:28.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:27 smithi145 bash[79878]: audit 2024-03-07T03:39:27.753838+0000 mon.a (mon.0) 1653 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:28.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:27 smithi145 bash[82329]: audit 2024-03-07T03:39:27.212861+0000 mon.a (mon.0) 1651 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:39:28.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:27 smithi145 bash[82329]: audit 2024-03-07T03:39:27.740988+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:28.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:27 smithi145 bash[82329]: audit 2024-03-07T03:39:27.753838+0000 mon.a (mon.0) 1653 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:28.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:27 smithi189 bash[66969]: audit 2024-03-07T03:39:27.212861+0000 mon.a (mon.0) 1651 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:39:28.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:27 smithi189 bash[66969]: audit 2024-03-07T03:39:27.740988+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:28.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:27 smithi189 bash[66969]: audit 2024-03-07T03:39:27.753838+0000 mon.a (mon.0) 1653 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:29.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:28 smithi145 bash[79878]: cluster 2024-03-07T03:39:27.444597+0000 mgr.y (mgr.34107) 828 : cluster 0 pgmap v1389: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 28 op/s 2024-03-07T03:39:29.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:28 smithi145 bash[82329]: cluster 2024-03-07T03:39:27.444597+0000 mgr.y (mgr.34107) 828 : cluster 0 pgmap v1389: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 28 op/s 2024-03-07T03:39:29.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:28 smithi189 bash[66969]: cluster 2024-03-07T03:39:27.444597+0000 mgr.y (mgr.34107) 828 : cluster 0 pgmap v1389: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 28 op/s 2024-03-07T03:39:31.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:30 smithi145 bash[79878]: cluster 2024-03-07T03:39:29.445534+0000 mgr.y (mgr.34107) 829 : cluster 0 pgmap v1390: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:39:31.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:30 smithi145 bash[79878]: cluster 2024-03-07T03:39:29.944146+0000 mon.a (mon.0) 1654 : cluster 0 osdmap e937: 8 total, 8 up, 8 in 2024-03-07T03:39:31.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:30 smithi145 bash[82329]: cluster 2024-03-07T03:39:29.445534+0000 mgr.y (mgr.34107) 829 : cluster 0 pgmap v1390: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:39:31.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:30 smithi145 bash[82329]: cluster 2024-03-07T03:39:29.944146+0000 mon.a (mon.0) 1654 : cluster 0 osdmap e937: 8 total, 8 up, 8 in 2024-03-07T03:39:31.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:30 smithi189 bash[66969]: cluster 2024-03-07T03:39:29.445534+0000 mgr.y (mgr.34107) 829 : cluster 0 pgmap v1390: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:39:31.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:30 smithi189 bash[66969]: cluster 2024-03-07T03:39:29.944146+0000 mon.a (mon.0) 1654 : cluster 0 osdmap e937: 8 total, 8 up, 8 in 2024-03-07T03:39:32.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:39:32 smithi189 bash[59814]: ts=2024-03-07T03:39:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:39:33.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:39:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:39:32] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2024-03-07T03:39:33.058 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:32 smithi145 bash[79878]: cluster 2024-03-07T03:39:31.446270+0000 mgr.y (mgr.34107) 830 : cluster 0 pgmap v1392: 105 pgs: 105 active+clean; 440 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.5 MiB/s wr, 31 op/s 2024-03-07T03:39:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:32 smithi145 bash[79878]: audit 2024-03-07T03:39:32.668131+0000 mon.a (mon.0) 1655 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:32 smithi145 bash[82329]: cluster 2024-03-07T03:39:31.446270+0000 mgr.y (mgr.34107) 830 : cluster 0 pgmap v1392: 105 pgs: 105 active+clean; 440 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.5 MiB/s wr, 31 op/s 2024-03-07T03:39:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:32 smithi145 bash[82329]: audit 2024-03-07T03:39:32.668131+0000 mon.a (mon.0) 1655 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:33.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:32 smithi189 bash[66969]: cluster 2024-03-07T03:39:31.446270+0000 mgr.y (mgr.34107) 830 : cluster 0 pgmap v1392: 105 pgs: 105 active+clean; 440 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.5 MiB/s wr, 31 op/s 2024-03-07T03:39:33.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:32 smithi189 bash[66969]: audit 2024-03-07T03:39:32.668131+0000 mon.a (mon.0) 1655 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:34.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:34 smithi145 bash[79878]: audit 2024-03-07T03:39:33.207099+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:34.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:34 smithi145 bash[79878]: audit 2024-03-07T03:39:33.215115+0000 mon.a (mon.0) 1657 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:34.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:34 smithi145 bash[79878]: cluster 2024-03-07T03:39:33.446887+0000 mgr.y (mgr.34107) 831 : cluster 0 pgmap v1393: 105 pgs: 105 active+clean; 440 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.5 MiB/s wr, 32 op/s 2024-03-07T03:39:34.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:34 smithi145 bash[79878]: audit 2024-03-07T03:39:33.846014+0000 mon.a (mon.0) 1658 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:34.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:34 smithi145 bash[79878]: audit 2024-03-07T03:39:33.851516+0000 mon.a (mon.0) 1659 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:34.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:34 smithi145 bash[82329]: audit 2024-03-07T03:39:33.207099+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:34.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:34 smithi145 bash[82329]: audit 2024-03-07T03:39:33.215115+0000 mon.a (mon.0) 1657 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:34.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:34 smithi145 bash[82329]: cluster 2024-03-07T03:39:33.446887+0000 mgr.y (mgr.34107) 831 : cluster 0 pgmap v1393: 105 pgs: 105 active+clean; 440 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.5 MiB/s wr, 32 op/s 2024-03-07T03:39:34.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:34 smithi145 bash[82329]: audit 2024-03-07T03:39:33.846014+0000 mon.a (mon.0) 1658 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:34.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:34 smithi145 bash[82329]: audit 2024-03-07T03:39:33.851516+0000 mon.a (mon.0) 1659 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:34.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:34 smithi189 bash[66969]: audit 2024-03-07T03:39:33.207099+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:34.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:34 smithi189 bash[66969]: audit 2024-03-07T03:39:33.215115+0000 mon.a (mon.0) 1657 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:34.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:34 smithi189 bash[66969]: cluster 2024-03-07T03:39:33.446887+0000 mgr.y (mgr.34107) 831 : cluster 0 pgmap v1393: 105 pgs: 105 active+clean; 440 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.5 MiB/s wr, 32 op/s 2024-03-07T03:39:34.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:34 smithi189 bash[66969]: audit 2024-03-07T03:39:33.846014+0000 mon.a (mon.0) 1658 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:34.579 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:34 smithi189 bash[66969]: audit 2024-03-07T03:39:33.851516+0000 mon.a (mon.0) 1659 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:35.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:35 smithi145 bash[79878]: audit 2024-03-07T03:39:34.426628+0000 mon.a (mon.0) 1660 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:39:35.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:35 smithi145 bash[79878]: audit 2024-03-07T03:39:34.427600+0000 mon.a (mon.0) 1661 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:39:35.559 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:35 smithi145 bash[79878]: audit 2024-03-07T03:39:34.433392+0000 mon.a (mon.0) 1662 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:35.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:35 smithi145 bash[82329]: audit 2024-03-07T03:39:34.426628+0000 mon.a (mon.0) 1660 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:39:35.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:35 smithi145 bash[82329]: audit 2024-03-07T03:39:34.427600+0000 mon.a (mon.0) 1661 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:39:35.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:35 smithi145 bash[82329]: audit 2024-03-07T03:39:34.433392+0000 mon.a (mon.0) 1662 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:35.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:35 smithi189 bash[66969]: audit 2024-03-07T03:39:34.426628+0000 mon.a (mon.0) 1660 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:39:35.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:35 smithi189 bash[66969]: audit 2024-03-07T03:39:34.427600+0000 mon.a (mon.0) 1661 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:39:35.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:35 smithi189 bash[66969]: audit 2024-03-07T03:39:34.433392+0000 mon.a (mon.0) 1662 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:36.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:36 smithi145 bash[79878]: cluster 2024-03-07T03:39:35.447605+0000 mgr.y (mgr.34107) 832 : cluster 0 pgmap v1394: 105 pgs: 105 active+clean; 452 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 4.5 MiB/s wr, 50 op/s 2024-03-07T03:39:36.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:36 smithi145 bash[82329]: cluster 2024-03-07T03:39:35.447605+0000 mgr.y (mgr.34107) 832 : cluster 0 pgmap v1394: 105 pgs: 105 active+clean; 452 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 4.5 MiB/s wr, 50 op/s 2024-03-07T03:39:36.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:36 smithi189 bash[66969]: cluster 2024-03-07T03:39:35.447605+0000 mgr.y (mgr.34107) 832 : cluster 0 pgmap v1394: 105 pgs: 105 active+clean; 452 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 4.5 MiB/s wr, 50 op/s 2024-03-07T03:39:38.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:38 smithi145 bash[79878]: cluster 2024-03-07T03:39:37.448543+0000 mgr.y (mgr.34107) 833 : cluster 0 pgmap v1395: 105 pgs: 105 active+clean; 456 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-03-07T03:39:38.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:38 smithi145 bash[82329]: cluster 2024-03-07T03:39:37.448543+0000 mgr.y (mgr.34107) 833 : cluster 0 pgmap v1395: 105 pgs: 105 active+clean; 456 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-03-07T03:39:38.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:38 smithi189 bash[66969]: cluster 2024-03-07T03:39:37.448543+0000 mgr.y (mgr.34107) 833 : cluster 0 pgmap v1395: 105 pgs: 105 active+clean; 456 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-03-07T03:39:40.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:40 smithi145 bash[79878]: cluster 2024-03-07T03:39:39.449308+0000 mgr.y (mgr.34107) 834 : cluster 0 pgmap v1396: 105 pgs: 105 active+clean; 464 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 4.1 MiB/s wr, 43 op/s 2024-03-07T03:39:40.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:40 smithi145 bash[82329]: cluster 2024-03-07T03:39:39.449308+0000 mgr.y (mgr.34107) 834 : cluster 0 pgmap v1396: 105 pgs: 105 active+clean; 464 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 4.1 MiB/s wr, 43 op/s 2024-03-07T03:39:40.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:40 smithi189 bash[66969]: cluster 2024-03-07T03:39:39.449308+0000 mgr.y (mgr.34107) 834 : cluster 0 pgmap v1396: 105 pgs: 105 active+clean; 464 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 4.1 MiB/s wr, 43 op/s 2024-03-07T03:39:40.960 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/4.ZeroOverlap (15709 ms) 2024-03-07T03:39:41.518 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/4 (128465 ms total) 2024-03-07T03:39:41.518 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:39:41.518 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/5, where TypeParam = UnformattedLUKS2 2024-03-07T03:39:41.576 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/5.Simple 2024-03-07T03:39:42.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:42 smithi189 bash[66969]: cluster 2024-03-07T03:39:41.450291+0000 mgr.y (mgr.34107) 835 : cluster 0 pgmap v1397: 105 pgs: 105 active+clean; 468 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 798 KiB/s rd, 3.8 MiB/s wr, 48 op/s 2024-03-07T03:39:42.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:42 smithi189 bash[66969]: cluster 2024-03-07T03:39:41.521565+0000 mon.a (mon.0) 1663 : cluster 0 osdmap e938: 8 total, 8 up, 8 in 2024-03-07T03:39:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:39:42 smithi189 bash[59814]: ts=2024-03-07T03:39:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:39:42.808 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:39:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:39:42] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2024-03-07T03:39:42.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:42 smithi145 bash[79878]: cluster 2024-03-07T03:39:41.450291+0000 mgr.y (mgr.34107) 835 : cluster 0 pgmap v1397: 105 pgs: 105 active+clean; 468 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 798 KiB/s rd, 3.8 MiB/s wr, 48 op/s 2024-03-07T03:39:42.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:42 smithi145 bash[79878]: cluster 2024-03-07T03:39:41.521565+0000 mon.a (mon.0) 1663 : cluster 0 osdmap e938: 8 total, 8 up, 8 in 2024-03-07T03:39:42.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:42 smithi145 bash[82329]: cluster 2024-03-07T03:39:41.450291+0000 mgr.y (mgr.34107) 835 : cluster 0 pgmap v1397: 105 pgs: 105 active+clean; 468 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 798 KiB/s rd, 3.8 MiB/s wr, 48 op/s 2024-03-07T03:39:42.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:42 smithi145 bash[82329]: cluster 2024-03-07T03:39:41.521565+0000 mon.a (mon.0) 1663 : cluster 0 osdmap e938: 8 total, 8 up, 8 in 2024-03-07T03:39:43.530 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:39:43.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:43 smithi145 bash[79878]: cluster 2024-03-07T03:39:42.544668+0000 mon.a (mon.0) 1664 : cluster 0 osdmap e939: 8 total, 8 up, 8 in 2024-03-07T03:39:43.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:43 smithi145 bash[79878]: audit 2024-03-07T03:39:42.547517+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.145:0/832302481' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:39:43.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:43 smithi145 bash[79878]: audit 2024-03-07T03:39:42.565342+0000 mon.a (mon.0) 1665 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:39:43.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:43 smithi145 bash[79878]: audit 2024-03-07T03:39:43.525316+0000 mon.a (mon.0) 1666 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-41","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:39:43.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:43 smithi145 bash[79878]: cluster 2024-03-07T03:39:43.534040+0000 mon.a (mon.0) 1667 : cluster 0 osdmap e940: 8 total, 8 up, 8 in 2024-03-07T03:39:43.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:43 smithi145 bash[82329]: cluster 2024-03-07T03:39:42.544668+0000 mon.a (mon.0) 1664 : cluster 0 osdmap e939: 8 total, 8 up, 8 in 2024-03-07T03:39:43.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:43 smithi145 bash[82329]: audit 2024-03-07T03:39:42.547517+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.145:0/832302481' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:39:43.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:43 smithi145 bash[82329]: audit 2024-03-07T03:39:42.565342+0000 mon.a (mon.0) 1665 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:39:43.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:43 smithi145 bash[82329]: audit 2024-03-07T03:39:43.525316+0000 mon.a (mon.0) 1666 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-41","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:39:43.810 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:43 smithi145 bash[82329]: cluster 2024-03-07T03:39:43.534040+0000 mon.a (mon.0) 1667 : cluster 0 osdmap e940: 8 total, 8 up, 8 in 2024-03-07T03:39:43.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:43 smithi189 bash[66969]: cluster 2024-03-07T03:39:42.544668+0000 mon.a (mon.0) 1664 : cluster 0 osdmap e939: 8 total, 8 up, 8 in 2024-03-07T03:39:43.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:43 smithi189 bash[66969]: audit 2024-03-07T03:39:42.547517+0000 mon.c (mon.2) 48 : audit 1 from='client.? 172.21.15.145:0/832302481' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:39:43.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:43 smithi189 bash[66969]: audit 2024-03-07T03:39:42.565342+0000 mon.a (mon.0) 1665 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:39:43.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:43 smithi189 bash[66969]: audit 2024-03-07T03:39:43.525316+0000 mon.a (mon.0) 1666 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-41","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:39:43.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:43 smithi189 bash[66969]: cluster 2024-03-07T03:39:43.534040+0000 mon.a (mon.0) 1667 : cluster 0 osdmap e940: 8 total, 8 up, 8 in 2024-03-07T03:39:44.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:44 smithi145 bash[79878]: cluster 2024-03-07T03:39:43.451012+0000 mgr.y (mgr.34107) 836 : cluster 0 pgmap v1400: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:39:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:44 smithi145 bash[79878]: cluster 2024-03-07T03:39:44.004310+0000 mon.a (mon.0) 1668 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:39:44.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:44 smithi145 bash[79878]: cluster 2024-03-07T03:39:44.535831+0000 mon.a (mon.0) 1669 : cluster 0 osdmap e941: 8 total, 8 up, 8 in 2024-03-07T03:39:44.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:44 smithi145 bash[82329]: cluster 2024-03-07T03:39:43.451012+0000 mgr.y (mgr.34107) 836 : cluster 0 pgmap v1400: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:39:44.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:44 smithi145 bash[82329]: cluster 2024-03-07T03:39:44.004310+0000 mon.a (mon.0) 1668 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:39:44.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:44 smithi145 bash[82329]: cluster 2024-03-07T03:39:44.535831+0000 mon.a (mon.0) 1669 : cluster 0 osdmap e941: 8 total, 8 up, 8 in 2024-03-07T03:39:44.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:44 smithi189 bash[66969]: cluster 2024-03-07T03:39:43.451012+0000 mgr.y (mgr.34107) 836 : cluster 0 pgmap v1400: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:39:44.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:44 smithi189 bash[66969]: cluster 2024-03-07T03:39:44.004310+0000 mon.a (mon.0) 1668 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:39:44.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:44 smithi189 bash[66969]: cluster 2024-03-07T03:39:44.535831+0000 mon.a (mon.0) 1669 : cluster 0 osdmap e941: 8 total, 8 up, 8 in 2024-03-07T03:39:46.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:46 smithi145 bash[79878]: cluster 2024-03-07T03:39:45.451712+0000 mgr.y (mgr.34107) 837 : cluster 0 pgmap v1403: 105 pgs: 18 unknown, 87 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:39:46.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:46 smithi145 bash[79878]: cluster 2024-03-07T03:39:45.535798+0000 mon.a (mon.0) 1670 : cluster 0 osdmap e942: 8 total, 8 up, 8 in 2024-03-07T03:39:46.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:46 smithi145 bash[79878]: cluster 2024-03-07T03:39:45.555796+0000 mon.a (mon.0) 1671 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:39:46.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:46 smithi145 bash[82329]: cluster 2024-03-07T03:39:45.451712+0000 mgr.y (mgr.34107) 837 : cluster 0 pgmap v1403: 105 pgs: 18 unknown, 87 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:39:46.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:46 smithi145 bash[82329]: cluster 2024-03-07T03:39:45.535798+0000 mon.a (mon.0) 1670 : cluster 0 osdmap e942: 8 total, 8 up, 8 in 2024-03-07T03:39:46.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:46 smithi145 bash[82329]: cluster 2024-03-07T03:39:45.555796+0000 mon.a (mon.0) 1671 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:39:46.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:46 smithi189 bash[66969]: cluster 2024-03-07T03:39:45.451712+0000 mgr.y (mgr.34107) 837 : cluster 0 pgmap v1403: 105 pgs: 18 unknown, 87 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:39:46.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:46 smithi189 bash[66969]: cluster 2024-03-07T03:39:45.535798+0000 mon.a (mon.0) 1670 : cluster 0 osdmap e942: 8 total, 8 up, 8 in 2024-03-07T03:39:46.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:46 smithi189 bash[66969]: cluster 2024-03-07T03:39:45.555796+0000 mon.a (mon.0) 1671 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:39:48.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:48 smithi145 bash[79878]: cluster 2024-03-07T03:39:47.452551+0000 mgr.y (mgr.34107) 838 : cluster 0 pgmap v1405: 105 pgs: 105 active+clean; 582 KiB data, 551 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.0 KiB/s wr, 3 op/s 2024-03-07T03:39:48.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:48 smithi145 bash[79878]: audit 2024-03-07T03:39:47.668592+0000 mon.a (mon.0) 1672 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:48.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:48 smithi145 bash[82329]: cluster 2024-03-07T03:39:47.452551+0000 mgr.y (mgr.34107) 838 : cluster 0 pgmap v1405: 105 pgs: 105 active+clean; 582 KiB data, 551 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.0 KiB/s wr, 3 op/s 2024-03-07T03:39:48.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:48 smithi145 bash[82329]: audit 2024-03-07T03:39:47.668592+0000 mon.a (mon.0) 1672 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:48.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:48 smithi189 bash[66969]: cluster 2024-03-07T03:39:47.452551+0000 mgr.y (mgr.34107) 838 : cluster 0 pgmap v1405: 105 pgs: 105 active+clean; 582 KiB data, 551 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.0 KiB/s wr, 3 op/s 2024-03-07T03:39:48.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:48 smithi189 bash[66969]: audit 2024-03-07T03:39:47.668592+0000 mon.a (mon.0) 1672 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:50.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:50 smithi145 bash[79878]: cluster 2024-03-07T03:39:49.453200+0000 mgr.y (mgr.34107) 839 : cluster 0 pgmap v1406: 105 pgs: 105 active+clean; 582 KiB data, 551 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-03-07T03:39:50.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:50 smithi145 bash[82329]: cluster 2024-03-07T03:39:49.453200+0000 mgr.y (mgr.34107) 839 : cluster 0 pgmap v1406: 105 pgs: 105 active+clean; 582 KiB data, 551 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-03-07T03:39:50.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:50 smithi189 bash[66969]: cluster 2024-03-07T03:39:49.453200+0000 mgr.y (mgr.34107) 839 : cluster 0 pgmap v1406: 105 pgs: 105 active+clean; 582 KiB data, 551 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-03-07T03:39:52.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:52 smithi189 bash[66969]: cluster 2024-03-07T03:39:51.453858+0000 mgr.y (mgr.34107) 840 : cluster 0 pgmap v1407: 105 pgs: 105 active+clean; 582 KiB data, 551 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-03-07T03:39:52.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:39:52 smithi189 bash[59814]: ts=2024-03-07T03:39:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:39:52.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:52 smithi145 bash[79878]: cluster 2024-03-07T03:39:51.453858+0000 mgr.y (mgr.34107) 840 : cluster 0 pgmap v1407: 105 pgs: 105 active+clean; 582 KiB data, 551 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-03-07T03:39:52.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:52 smithi145 bash[82329]: cluster 2024-03-07T03:39:51.453858+0000 mgr.y (mgr.34107) 840 : cluster 0 pgmap v1407: 105 pgs: 105 active+clean; 582 KiB data, 551 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-03-07T03:39:52.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:39:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:39:52] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.43.0" 2024-03-07T03:39:54.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:54 smithi145 bash[79878]: cluster 2024-03-07T03:39:53.006088+0000 mon.a (mon.0) 1673 : cluster 0 osdmap e943: 8 total, 8 up, 8 in 2024-03-07T03:39:54.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:54 smithi145 bash[82329]: cluster 2024-03-07T03:39:53.006088+0000 mon.a (mon.0) 1673 : cluster 0 osdmap e943: 8 total, 8 up, 8 in 2024-03-07T03:39:54.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:54 smithi189 bash[66969]: cluster 2024-03-07T03:39:53.006088+0000 mon.a (mon.0) 1673 : cluster 0 osdmap e943: 8 total, 8 up, 8 in 2024-03-07T03:39:55.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:55 smithi145 bash[79878]: cluster 2024-03-07T03:39:53.454552+0000 mgr.y (mgr.34107) 841 : cluster 0 pgmap v1409: 105 pgs: 105 active+clean; 582 KiB data, 551 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-03-07T03:39:55.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:55 smithi145 bash[79878]: cluster 2024-03-07T03:39:54.016253+0000 mon.a (mon.0) 1674 : cluster 0 osdmap e944: 8 total, 8 up, 8 in 2024-03-07T03:39:55.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:55 smithi145 bash[79878]: audit 2024-03-07T03:39:54.625485+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:55.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:55 smithi145 bash[82329]: cluster 2024-03-07T03:39:53.454552+0000 mgr.y (mgr.34107) 841 : cluster 0 pgmap v1409: 105 pgs: 105 active+clean; 582 KiB data, 551 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-03-07T03:39:55.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:55 smithi145 bash[82329]: cluster 2024-03-07T03:39:54.016253+0000 mon.a (mon.0) 1674 : cluster 0 osdmap e944: 8 total, 8 up, 8 in 2024-03-07T03:39:55.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:55 smithi145 bash[82329]: audit 2024-03-07T03:39:54.625485+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:55.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:55 smithi189 bash[66969]: cluster 2024-03-07T03:39:53.454552+0000 mgr.y (mgr.34107) 841 : cluster 0 pgmap v1409: 105 pgs: 105 active+clean; 582 KiB data, 551 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-03-07T03:39:55.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:55 smithi189 bash[66969]: cluster 2024-03-07T03:39:54.016253+0000 mon.a (mon.0) 1674 : cluster 0 osdmap e944: 8 total, 8 up, 8 in 2024-03-07T03:39:55.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:55 smithi189 bash[66969]: audit 2024-03-07T03:39:54.625485+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:39:57.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:57 smithi145 bash[79878]: cluster 2024-03-07T03:39:55.455156+0000 mgr.y (mgr.34107) 842 : cluster 0 pgmap v1411: 105 pgs: 105 active+clean; 29 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.5 MiB/s wr, 7 op/s 2024-03-07T03:39:57.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:57 smithi145 bash[82329]: cluster 2024-03-07T03:39:55.455156+0000 mgr.y (mgr.34107) 842 : cluster 0 pgmap v1411: 105 pgs: 105 active+clean; 29 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.5 MiB/s wr, 7 op/s 2024-03-07T03:39:57.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:57 smithi189 bash[66969]: cluster 2024-03-07T03:39:55.455156+0000 mgr.y (mgr.34107) 842 : cluster 0 pgmap v1411: 105 pgs: 105 active+clean; 29 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.5 MiB/s wr, 7 op/s 2024-03-07T03:39:59.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:39:59 smithi145 bash[79878]: cluster 2024-03-07T03:39:57.455858+0000 mgr.y (mgr.34107) 843 : cluster 0 pgmap v1412: 105 pgs: 105 active+clean; 42 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.1 MiB/s wr, 20 op/s 2024-03-07T03:39:59.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:39:59 smithi145 bash[82329]: cluster 2024-03-07T03:39:57.455858+0000 mgr.y (mgr.34107) 843 : cluster 0 pgmap v1412: 105 pgs: 105 active+clean; 42 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.1 MiB/s wr, 20 op/s 2024-03-07T03:39:59.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:39:59 smithi189 bash[66969]: cluster 2024-03-07T03:39:57.455858+0000 mgr.y (mgr.34107) 843 : cluster 0 pgmap v1412: 105 pgs: 105 active+clean; 42 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.1 MiB/s wr, 20 op/s 2024-03-07T03:39:59.994 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/5.Simple (18417 ms) 2024-03-07T03:39:59.994 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/5.Grow 2024-03-07T03:39:59.994 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:40:00.308 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:00 smithi145 bash[82329]: cluster 2024-03-07T03:40:00.000126+0000 mon.a (mon.0) 1676 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:40:00.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:00 smithi145 bash[79878]: debug 2024-03-07T03:39:59.996+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:40:00.309 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:00 smithi145 bash[79878]: cluster 2024-03-07T03:40:00.000126+0000 mon.a (mon.0) 1676 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:40:00.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:00 smithi189 bash[66969]: cluster 2024-03-07T03:40:00.000126+0000 mon.a (mon.0) 1676 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:40:01.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:01 smithi145 bash[79878]: cluster 2024-03-07T03:39:59.456516+0000 mgr.y (mgr.34107) 844 : cluster 0 pgmap v1413: 105 pgs: 105 active+clean; 54 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 6.6 MiB/s wr, 60 op/s 2024-03-07T03:40:01.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:01 smithi145 bash[82329]: cluster 2024-03-07T03:39:59.456516+0000 mgr.y (mgr.34107) 844 : cluster 0 pgmap v1413: 105 pgs: 105 active+clean; 54 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 6.6 MiB/s wr, 60 op/s 2024-03-07T03:40:01.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:01 smithi189 bash[66969]: cluster 2024-03-07T03:39:59.456516+0000 mgr.y (mgr.34107) 844 : cluster 0 pgmap v1413: 105 pgs: 105 active+clean; 54 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 6.6 MiB/s wr, 60 op/s 2024-03-07T03:40:02.578 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:40:02 smithi189 bash[59814]: ts=2024-03-07T03:40:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:40:03.043 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:40:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:40:02] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.43.0" 2024-03-07T03:40:03.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:03 smithi145 bash[79878]: cluster 2024-03-07T03:40:01.457270+0000 mgr.y (mgr.34107) 845 : cluster 0 pgmap v1414: 105 pgs: 105 active+clean; 78 MiB data, 772 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 9.1 MiB/s wr, 62 op/s 2024-03-07T03:40:03.308 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:03 smithi145 bash[79878]: audit 2024-03-07T03:40:02.668751+0000 mon.a (mon.0) 1677 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:03.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:03 smithi145 bash[82329]: cluster 2024-03-07T03:40:01.457270+0000 mgr.y (mgr.34107) 845 : cluster 0 pgmap v1414: 105 pgs: 105 active+clean; 78 MiB data, 772 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 9.1 MiB/s wr, 62 op/s 2024-03-07T03:40:03.309 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:03 smithi145 bash[82329]: audit 2024-03-07T03:40:02.668751+0000 mon.a (mon.0) 1677 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:03.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:03 smithi189 bash[66969]: cluster 2024-03-07T03:40:01.457270+0000 mgr.y (mgr.34107) 845 : cluster 0 pgmap v1414: 105 pgs: 105 active+clean; 78 MiB data, 772 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 9.1 MiB/s wr, 62 op/s 2024-03-07T03:40:03.328 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:03 smithi189 bash[66969]: audit 2024-03-07T03:40:02.668751+0000 mon.a (mon.0) 1677 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:04.558 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:04 smithi145 bash[79878]: cluster 2024-03-07T03:40:03.457910+0000 mgr.y (mgr.34107) 846 : cluster 0 pgmap v1415: 105 pgs: 105 active+clean; 83 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 8.2 MiB/s wr, 65 op/s 2024-03-07T03:40:04.559 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:04 smithi145 bash[82329]: cluster 2024-03-07T03:40:03.457910+0000 mgr.y (mgr.34107) 846 : cluster 0 pgmap v1415: 105 pgs: 105 active+clean; 83 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 8.2 MiB/s wr, 65 op/s 2024-03-07T03:40:04.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:04 smithi189 bash[66969]: cluster 2024-03-07T03:40:03.457910+0000 mgr.y (mgr.34107) 846 : cluster 0 pgmap v1415: 105 pgs: 105 active+clean; 83 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 8.2 MiB/s wr, 65 op/s 2024-03-07T03:40:06.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:06 smithi145 bash[79878]: cluster 2024-03-07T03:40:05.458630+0000 mgr.y (mgr.34107) 847 : cluster 0 pgmap v1416: 105 pgs: 105 active+clean; 83 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.1 MiB/s wr, 57 op/s 2024-03-07T03:40:06.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:06 smithi145 bash[82329]: cluster 2024-03-07T03:40:05.458630+0000 mgr.y (mgr.34107) 847 : cluster 0 pgmap v1416: 105 pgs: 105 active+clean; 83 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.1 MiB/s wr, 57 op/s 2024-03-07T03:40:06.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:06 smithi189 bash[66969]: cluster 2024-03-07T03:40:05.458630+0000 mgr.y (mgr.34107) 847 : cluster 0 pgmap v1416: 105 pgs: 105 active+clean; 83 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.1 MiB/s wr, 57 op/s 2024-03-07T03:40:08.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:08 smithi145 bash[79878]: cluster 2024-03-07T03:40:07.459528+0000 mgr.y (mgr.34107) 848 : cluster 0 pgmap v1417: 105 pgs: 105 active+clean; 83 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.5 MiB/s wr, 55 op/s 2024-03-07T03:40:08.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:08 smithi145 bash[79878]: cluster 2024-03-07T03:40:07.535032+0000 mon.a (mon.0) 1678 : cluster 0 osdmap e945: 8 total, 8 up, 8 in 2024-03-07T03:40:08.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:08 smithi145 bash[82329]: cluster 2024-03-07T03:40:07.459528+0000 mgr.y (mgr.34107) 848 : cluster 0 pgmap v1417: 105 pgs: 105 active+clean; 83 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.5 MiB/s wr, 55 op/s 2024-03-07T03:40:08.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:08 smithi145 bash[82329]: cluster 2024-03-07T03:40:07.535032+0000 mon.a (mon.0) 1678 : cluster 0 osdmap e945: 8 total, 8 up, 8 in 2024-03-07T03:40:08.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:08 smithi189 bash[66969]: cluster 2024-03-07T03:40:07.459528+0000 mgr.y (mgr.34107) 848 : cluster 0 pgmap v1417: 105 pgs: 105 active+clean; 83 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.5 MiB/s wr, 55 op/s 2024-03-07T03:40:08.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:08 smithi189 bash[66969]: cluster 2024-03-07T03:40:07.535032+0000 mon.a (mon.0) 1678 : cluster 0 osdmap e945: 8 total, 8 up, 8 in 2024-03-07T03:40:10.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:10 smithi145 bash[79878]: cluster 2024-03-07T03:40:09.460476+0000 mgr.y (mgr.34107) 849 : cluster 0 pgmap v1419: 105 pgs: 105 active+clean; 103 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 4.9 MiB/s wr, 20 op/s 2024-03-07T03:40:10.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:10 smithi145 bash[82329]: cluster 2024-03-07T03:40:09.460476+0000 mgr.y (mgr.34107) 849 : cluster 0 pgmap v1419: 105 pgs: 105 active+clean; 103 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 4.9 MiB/s wr, 20 op/s 2024-03-07T03:40:10.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:10 smithi189 bash[66969]: cluster 2024-03-07T03:40:09.460476+0000 mgr.y (mgr.34107) 849 : cluster 0 pgmap v1419: 105 pgs: 105 active+clean; 103 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 4.9 MiB/s wr, 20 op/s 2024-03-07T03:40:12.578 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:12 smithi189 bash[66969]: cluster 2024-03-07T03:40:11.462597+0000 mgr.y (mgr.34107) 850 : cluster 0 pgmap v1420: 105 pgs: 105 active+clean; 116 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.8 MiB/s wr, 37 op/s 2024-03-07T03:40:12.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:40:12 smithi189 bash[59814]: ts=2024-03-07T03:40:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:40:12.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:12 smithi145 bash[79878]: cluster 2024-03-07T03:40:11.462597+0000 mgr.y (mgr.34107) 850 : cluster 0 pgmap v1420: 105 pgs: 105 active+clean; 116 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.8 MiB/s wr, 37 op/s 2024-03-07T03:40:12.809 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:40:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:40:12] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.43.0" 2024-03-07T03:40:12.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:12 smithi145 bash[82329]: cluster 2024-03-07T03:40:11.462597+0000 mgr.y (mgr.34107) 850 : cluster 0 pgmap v1420: 105 pgs: 105 active+clean; 116 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.8 MiB/s wr, 37 op/s 2024-03-07T03:40:13.554 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/5.Grow (13560 ms) 2024-03-07T03:40:13.554 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/5.Shrink 2024-03-07T03:40:13.554 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:40:14.808 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:14 smithi145 bash[79878]: cluster 2024-03-07T03:40:13.463438+0000 mgr.y (mgr.34107) 851 : cluster 0 pgmap v1421: 105 pgs: 105 active+clean; 133 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.0 MiB/s wr, 53 op/s 2024-03-07T03:40:14.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:14 smithi145 bash[82329]: cluster 2024-03-07T03:40:13.463438+0000 mgr.y (mgr.34107) 851 : cluster 0 pgmap v1421: 105 pgs: 105 active+clean; 133 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.0 MiB/s wr, 53 op/s 2024-03-07T03:40:14.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:14 smithi189 bash[66969]: cluster 2024-03-07T03:40:13.463438+0000 mgr.y (mgr.34107) 851 : cluster 0 pgmap v1421: 105 pgs: 105 active+clean; 133 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.0 MiB/s wr, 53 op/s 2024-03-07T03:40:16.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:16 smithi145 bash[79878]: cluster 2024-03-07T03:40:15.464212+0000 mgr.y (mgr.34107) 852 : cluster 0 pgmap v1422: 105 pgs: 105 active+clean; 157 MiB data, 981 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 7.4 MiB/s wr, 66 op/s 2024-03-07T03:40:16.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:16 smithi145 bash[82329]: cluster 2024-03-07T03:40:15.464212+0000 mgr.y (mgr.34107) 852 : cluster 0 pgmap v1422: 105 pgs: 105 active+clean; 157 MiB data, 981 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 7.4 MiB/s wr, 66 op/s 2024-03-07T03:40:16.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:16 smithi189 bash[66969]: cluster 2024-03-07T03:40:15.464212+0000 mgr.y (mgr.34107) 852 : cluster 0 pgmap v1422: 105 pgs: 105 active+clean; 157 MiB data, 981 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 7.4 MiB/s wr, 66 op/s 2024-03-07T03:40:18.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:18 smithi145 bash[79878]: cluster 2024-03-07T03:40:17.464925+0000 mgr.y (mgr.34107) 853 : cluster 0 pgmap v1423: 105 pgs: 105 active+clean; 165 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 70 op/s 2024-03-07T03:40:18.809 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:18 smithi145 bash[79878]: audit 2024-03-07T03:40:17.669050+0000 mon.a (mon.0) 1679 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:18.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:18 smithi145 bash[82329]: cluster 2024-03-07T03:40:17.464925+0000 mgr.y (mgr.34107) 853 : cluster 0 pgmap v1423: 105 pgs: 105 active+clean; 165 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 70 op/s 2024-03-07T03:40:18.809 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:18 smithi145 bash[82329]: audit 2024-03-07T03:40:17.669050+0000 mon.a (mon.0) 1679 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:18.828 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:18 smithi189 bash[66969]: cluster 2024-03-07T03:40:17.464925+0000 mgr.y (mgr.34107) 853 : cluster 0 pgmap v1423: 105 pgs: 105 active+clean; 165 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 70 op/s 2024-03-07T03:40:18.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:18 smithi189 bash[66969]: audit 2024-03-07T03:40:17.669050+0000 mon.a (mon.0) 1679 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:20.829 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:20 smithi189 bash[66969]: cluster 2024-03-07T03:40:19.465835+0000 mgr.y (mgr.34107) 854 : cluster 0 pgmap v1424: 105 pgs: 105 active+clean; 165 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 6.9 MiB/s wr, 59 op/s 2024-03-07T03:40:21.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:20 smithi145 bash[79878]: cluster 2024-03-07T03:40:19.465835+0000 mgr.y (mgr.34107) 854 : cluster 0 pgmap v1424: 105 pgs: 105 active+clean; 165 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 6.9 MiB/s wr, 59 op/s 2024-03-07T03:40:21.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:20 smithi145 bash[82329]: cluster 2024-03-07T03:40:19.465835+0000 mgr.y (mgr.34107) 854 : cluster 0 pgmap v1424: 105 pgs: 105 active+clean; 165 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 6.9 MiB/s wr, 59 op/s 2024-03-07T03:40:22.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:21 smithi145 bash[79878]: cluster 2024-03-07T03:40:20.580792+0000 mon.a (mon.0) 1680 : cluster 0 osdmap e946: 8 total, 8 up, 8 in 2024-03-07T03:40:22.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:21 smithi145 bash[82329]: cluster 2024-03-07T03:40:20.580792+0000 mon.a (mon.0) 1680 : cluster 0 osdmap e946: 8 total, 8 up, 8 in 2024-03-07T03:40:22.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:21 smithi189 bash[66969]: cluster 2024-03-07T03:40:20.580792+0000 mon.a (mon.0) 1680 : cluster 0 osdmap e946: 8 total, 8 up, 8 in 2024-03-07T03:40:22.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:40:22 smithi189 bash[59814]: ts=2024-03-07T03:40:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:40:23.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:22 smithi145 bash[79878]: cluster 2024-03-07T03:40:21.466560+0000 mgr.y (mgr.34107) 855 : cluster 0 pgmap v1426: 105 pgs: 105 active+clean; 182 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.6 MiB/s wr, 63 op/s 2024-03-07T03:40:23.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:40:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:40:22] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2024-03-07T03:40:23.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:22 smithi145 bash[82329]: cluster 2024-03-07T03:40:21.466560+0000 mgr.y (mgr.34107) 855 : cluster 0 pgmap v1426: 105 pgs: 105 active+clean; 182 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.6 MiB/s wr, 63 op/s 2024-03-07T03:40:23.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:22 smithi189 bash[66969]: cluster 2024-03-07T03:40:21.466560+0000 mgr.y (mgr.34107) 855 : cluster 0 pgmap v1426: 105 pgs: 105 active+clean; 182 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.6 MiB/s wr, 63 op/s 2024-03-07T03:40:25.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:24 smithi145 bash[79878]: cluster 2024-03-07T03:40:23.467173+0000 mgr.y (mgr.34107) 856 : cluster 0 pgmap v1427: 105 pgs: 105 active+clean; 182 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.9 MiB/s wr, 38 op/s 2024-03-07T03:40:25.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:24 smithi145 bash[82329]: cluster 2024-03-07T03:40:23.467173+0000 mgr.y (mgr.34107) 856 : cluster 0 pgmap v1427: 105 pgs: 105 active+clean; 182 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.9 MiB/s wr, 38 op/s 2024-03-07T03:40:25.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:24 smithi189 bash[66969]: cluster 2024-03-07T03:40:23.467173+0000 mgr.y (mgr.34107) 856 : cluster 0 pgmap v1427: 105 pgs: 105 active+clean; 182 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.9 MiB/s wr, 38 op/s 2024-03-07T03:40:26.629 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/5.Shrink (13075 ms) 2024-03-07T03:40:26.630 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/5.ShrinkToOne 2024-03-07T03:40:26.630 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:40:27.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:26 smithi145 bash[79878]: cluster 2024-03-07T03:40:25.467858+0000 mgr.y (mgr.34107) 857 : cluster 0 pgmap v1428: 105 pgs: 105 active+clean; 206 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.9 MiB/s wr, 40 op/s 2024-03-07T03:40:27.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:26 smithi145 bash[82329]: cluster 2024-03-07T03:40:25.467858+0000 mgr.y (mgr.34107) 857 : cluster 0 pgmap v1428: 105 pgs: 105 active+clean; 206 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.9 MiB/s wr, 40 op/s 2024-03-07T03:40:27.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:26 smithi189 bash[66969]: cluster 2024-03-07T03:40:25.467858+0000 mgr.y (mgr.34107) 857 : cluster 0 pgmap v1428: 105 pgs: 105 active+clean; 206 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.9 MiB/s wr, 40 op/s 2024-03-07T03:40:29.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:28 smithi145 bash[79878]: cluster 2024-03-07T03:40:27.468863+0000 mgr.y (mgr.34107) 858 : cluster 0 pgmap v1429: 105 pgs: 105 active+clean; 235 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 7.0 MiB/s wr, 63 op/s 2024-03-07T03:40:29.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:28 smithi145 bash[82329]: cluster 2024-03-07T03:40:27.468863+0000 mgr.y (mgr.34107) 858 : cluster 0 pgmap v1429: 105 pgs: 105 active+clean; 235 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 7.0 MiB/s wr, 63 op/s 2024-03-07T03:40:29.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:28 smithi189 bash[66969]: cluster 2024-03-07T03:40:27.468863+0000 mgr.y (mgr.34107) 858 : cluster 0 pgmap v1429: 105 pgs: 105 active+clean; 235 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 7.0 MiB/s wr, 63 op/s 2024-03-07T03:40:31.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:30 smithi145 bash[79878]: cluster 2024-03-07T03:40:29.469536+0000 mgr.y (mgr.34107) 859 : cluster 0 pgmap v1430: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 8.2 MiB/s wr, 70 op/s 2024-03-07T03:40:31.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:30 smithi145 bash[82329]: cluster 2024-03-07T03:40:29.469536+0000 mgr.y (mgr.34107) 859 : cluster 0 pgmap v1430: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 8.2 MiB/s wr, 70 op/s 2024-03-07T03:40:31.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:30 smithi189 bash[66969]: cluster 2024-03-07T03:40:29.469536+0000 mgr.y (mgr.34107) 859 : cluster 0 pgmap v1430: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 8.2 MiB/s wr, 70 op/s 2024-03-07T03:40:32.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:40:32 smithi189 bash[59814]: ts=2024-03-07T03:40:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:40:33.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:32 smithi145 bash[79878]: cluster 2024-03-07T03:40:31.470218+0000 mgr.y (mgr.34107) 860 : cluster 0 pgmap v1431: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.5 MiB/s wr, 63 op/s 2024-03-07T03:40:33.059 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:40:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:40:32] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.43.0" 2024-03-07T03:40:33.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:32 smithi145 bash[82329]: cluster 2024-03-07T03:40:31.470218+0000 mgr.y (mgr.34107) 860 : cluster 0 pgmap v1431: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.5 MiB/s wr, 63 op/s 2024-03-07T03:40:33.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:32 smithi189 bash[66969]: cluster 2024-03-07T03:40:31.470218+0000 mgr.y (mgr.34107) 860 : cluster 0 pgmap v1431: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.5 MiB/s wr, 63 op/s 2024-03-07T03:40:34.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:33 smithi145 bash[79878]: audit 2024-03-07T03:40:32.669285+0000 mon.a (mon.0) 1681 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:34.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:33 smithi145 bash[82329]: audit 2024-03-07T03:40:32.669285+0000 mon.a (mon.0) 1681 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:34.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:33 smithi189 bash[66969]: audit 2024-03-07T03:40:32.669285+0000 mon.a (mon.0) 1681 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:35.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:34 smithi145 bash[79878]: cluster 2024-03-07T03:40:33.471026+0000 mgr.y (mgr.34107) 861 : cluster 0 pgmap v1432: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 5.4 MiB/s wr, 47 op/s 2024-03-07T03:40:35.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:34 smithi145 bash[79878]: cluster 2024-03-07T03:40:33.665353+0000 mon.a (mon.0) 1682 : cluster 0 osdmap e947: 8 total, 8 up, 8 in 2024-03-07T03:40:35.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:34 smithi145 bash[79878]: audit 2024-03-07T03:40:34.514358+0000 mon.a (mon.0) 1683 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:40:35.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:34 smithi145 bash[82329]: cluster 2024-03-07T03:40:33.471026+0000 mgr.y (mgr.34107) 861 : cluster 0 pgmap v1432: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 5.4 MiB/s wr, 47 op/s 2024-03-07T03:40:35.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:34 smithi145 bash[82329]: cluster 2024-03-07T03:40:33.665353+0000 mon.a (mon.0) 1682 : cluster 0 osdmap e947: 8 total, 8 up, 8 in 2024-03-07T03:40:35.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:34 smithi145 bash[82329]: audit 2024-03-07T03:40:34.514358+0000 mon.a (mon.0) 1683 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:40:35.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:34 smithi189 bash[66969]: cluster 2024-03-07T03:40:33.471026+0000 mgr.y (mgr.34107) 861 : cluster 0 pgmap v1432: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 5.4 MiB/s wr, 47 op/s 2024-03-07T03:40:35.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:34 smithi189 bash[66969]: cluster 2024-03-07T03:40:33.665353+0000 mon.a (mon.0) 1682 : cluster 0 osdmap e947: 8 total, 8 up, 8 in 2024-03-07T03:40:35.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:34 smithi189 bash[66969]: audit 2024-03-07T03:40:34.514358+0000 mon.a (mon.0) 1683 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:40:37.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:36 smithi145 bash[79878]: cluster 2024-03-07T03:40:35.471745+0000 mgr.y (mgr.34107) 862 : cluster 0 pgmap v1434: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.4 MiB/s wr, 52 op/s 2024-03-07T03:40:37.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:36 smithi145 bash[82329]: cluster 2024-03-07T03:40:35.471745+0000 mgr.y (mgr.34107) 862 : cluster 0 pgmap v1434: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.4 MiB/s wr, 52 op/s 2024-03-07T03:40:37.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:36 smithi189 bash[66969]: cluster 2024-03-07T03:40:35.471745+0000 mgr.y (mgr.34107) 862 : cluster 0 pgmap v1434: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.4 MiB/s wr, 52 op/s 2024-03-07T03:40:39.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:38 smithi145 bash[79878]: cluster 2024-03-07T03:40:37.472733+0000 mgr.y (mgr.34107) 863 : cluster 0 pgmap v1435: 105 pgs: 105 active+clean; 288 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 5.3 MiB/s wr, 38 op/s 2024-03-07T03:40:39.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:38 smithi145 bash[82329]: cluster 2024-03-07T03:40:37.472733+0000 mgr.y (mgr.34107) 863 : cluster 0 pgmap v1435: 105 pgs: 105 active+clean; 288 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 5.3 MiB/s wr, 38 op/s 2024-03-07T03:40:39.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:38 smithi189 bash[66969]: cluster 2024-03-07T03:40:37.472733+0000 mgr.y (mgr.34107) 863 : cluster 0 pgmap v1435: 105 pgs: 105 active+clean; 288 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 5.3 MiB/s wr, 38 op/s 2024-03-07T03:40:39.553 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/5.ShrinkToOne (12923 ms) 2024-03-07T03:40:39.553 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/5.ShrinkToOneAfterSnapshot 2024-03-07T03:40:39.553 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:40:41.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:40 smithi145 bash[79878]: cluster 2024-03-07T03:40:39.473684+0000 mgr.y (mgr.34107) 864 : cluster 0 pgmap v1436: 105 pgs: 105 active+clean; 300 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 5.7 MiB/s wr, 49 op/s 2024-03-07T03:40:41.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:40 smithi145 bash[79878]: audit 2024-03-07T03:40:40.298900+0000 mon.a (mon.0) 1684 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:41.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:40 smithi145 bash[79878]: audit 2024-03-07T03:40:40.310475+0000 mon.a (mon.0) 1685 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:41.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:40 smithi145 bash[79878]: audit 2024-03-07T03:40:40.501734+0000 mon.a (mon.0) 1686 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:41.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:40 smithi145 bash[79878]: audit 2024-03-07T03:40:40.508570+0000 mon.a (mon.0) 1687 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:41.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:40 smithi145 bash[82329]: cluster 2024-03-07T03:40:39.473684+0000 mgr.y (mgr.34107) 864 : cluster 0 pgmap v1436: 105 pgs: 105 active+clean; 300 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 5.7 MiB/s wr, 49 op/s 2024-03-07T03:40:41.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:40 smithi145 bash[82329]: audit 2024-03-07T03:40:40.298900+0000 mon.a (mon.0) 1684 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:41.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:40 smithi145 bash[82329]: audit 2024-03-07T03:40:40.310475+0000 mon.a (mon.0) 1685 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:41.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:40 smithi145 bash[82329]: audit 2024-03-07T03:40:40.501734+0000 mon.a (mon.0) 1686 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:41.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:40 smithi145 bash[82329]: audit 2024-03-07T03:40:40.508570+0000 mon.a (mon.0) 1687 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:40 smithi189 bash[66969]: cluster 2024-03-07T03:40:39.473684+0000 mgr.y (mgr.34107) 864 : cluster 0 pgmap v1436: 105 pgs: 105 active+clean; 300 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 5.7 MiB/s wr, 49 op/s 2024-03-07T03:40:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:40 smithi189 bash[66969]: audit 2024-03-07T03:40:40.298900+0000 mon.a (mon.0) 1684 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:40 smithi189 bash[66969]: audit 2024-03-07T03:40:40.310475+0000 mon.a (mon.0) 1685 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:40 smithi189 bash[66969]: audit 2024-03-07T03:40:40.501734+0000 mon.a (mon.0) 1686 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:41.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:40 smithi189 bash[66969]: audit 2024-03-07T03:40:40.508570+0000 mon.a (mon.0) 1687 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:42.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:41 smithi145 bash[79878]: audit 2024-03-07T03:40:41.078732+0000 mon.a (mon.0) 1688 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:40:42.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:41 smithi145 bash[79878]: audit 2024-03-07T03:40:41.079842+0000 mon.a (mon.0) 1689 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:40:42.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:41 smithi145 bash[79878]: audit 2024-03-07T03:40:41.087192+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:42.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:41 smithi145 bash[82329]: audit 2024-03-07T03:40:41.078732+0000 mon.a (mon.0) 1688 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:40:42.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:41 smithi145 bash[82329]: audit 2024-03-07T03:40:41.079842+0000 mon.a (mon.0) 1689 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:40:42.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:41 smithi145 bash[82329]: audit 2024-03-07T03:40:41.087192+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:42.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:41 smithi189 bash[66969]: audit 2024-03-07T03:40:41.078732+0000 mon.a (mon.0) 1688 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:40:42.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:41 smithi189 bash[66969]: audit 2024-03-07T03:40:41.079842+0000 mon.a (mon.0) 1689 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:40:42.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:41 smithi189 bash[66969]: audit 2024-03-07T03:40:41.087192+0000 mon.a (mon.0) 1690 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:40:42.579 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:40:42 smithi189 bash[59814]: ts=2024-03-07T03:40:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:40:43.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:42 smithi145 bash[79878]: cluster 2024-03-07T03:40:41.474425+0000 mgr.y (mgr.34107) 865 : cluster 0 pgmap v1437: 105 pgs: 105 active+clean; 300 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 5.7 MiB/s wr, 49 op/s 2024-03-07T03:40:43.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:42 smithi145 bash[82329]: cluster 2024-03-07T03:40:41.474425+0000 mgr.y (mgr.34107) 865 : cluster 0 pgmap v1437: 105 pgs: 105 active+clean; 300 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 5.7 MiB/s wr, 49 op/s 2024-03-07T03:40:43.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:40:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:40:42] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.43.0" 2024-03-07T03:40:43.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:42 smithi189 bash[66969]: cluster 2024-03-07T03:40:41.474425+0000 mgr.y (mgr.34107) 865 : cluster 0 pgmap v1437: 105 pgs: 105 active+clean; 300 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 5.7 MiB/s wr, 49 op/s 2024-03-07T03:40:45.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:44 smithi145 bash[79878]: cluster 2024-03-07T03:40:43.475042+0000 mgr.y (mgr.34107) 866 : cluster 0 pgmap v1438: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 61 op/s 2024-03-07T03:40:45.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:44 smithi145 bash[82329]: cluster 2024-03-07T03:40:43.475042+0000 mgr.y (mgr.34107) 866 : cluster 0 pgmap v1438: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 61 op/s 2024-03-07T03:40:45.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:44 smithi189 bash[66969]: cluster 2024-03-07T03:40:43.475042+0000 mgr.y (mgr.34107) 866 : cluster 0 pgmap v1438: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 61 op/s 2024-03-07T03:40:47.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:46 smithi145 bash[79878]: cluster 2024-03-07T03:40:45.475736+0000 mgr.y (mgr.34107) 867 : cluster 0 pgmap v1439: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.5 MiB/s wr, 52 op/s 2024-03-07T03:40:47.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:46 smithi145 bash[82329]: cluster 2024-03-07T03:40:45.475736+0000 mgr.y (mgr.34107) 867 : cluster 0 pgmap v1439: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.5 MiB/s wr, 52 op/s 2024-03-07T03:40:47.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:46 smithi189 bash[66969]: cluster 2024-03-07T03:40:45.475736+0000 mgr.y (mgr.34107) 867 : cluster 0 pgmap v1439: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.5 MiB/s wr, 52 op/s 2024-03-07T03:40:48.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:47 smithi145 bash[79878]: audit 2024-03-07T03:40:47.669622+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:48.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:47 smithi145 bash[82329]: audit 2024-03-07T03:40:47.669622+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:48.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:47 smithi189 bash[66969]: audit 2024-03-07T03:40:47.669622+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:49.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:48 smithi145 bash[79878]: cluster 2024-03-07T03:40:47.476449+0000 mgr.y (mgr.34107) 868 : cluster 0 pgmap v1440: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 4.3 MiB/s wr, 37 op/s 2024-03-07T03:40:49.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:48 smithi145 bash[82329]: cluster 2024-03-07T03:40:47.476449+0000 mgr.y (mgr.34107) 868 : cluster 0 pgmap v1440: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 4.3 MiB/s wr, 37 op/s 2024-03-07T03:40:49.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:48 smithi189 bash[66969]: cluster 2024-03-07T03:40:47.476449+0000 mgr.y (mgr.34107) 868 : cluster 0 pgmap v1440: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 4.3 MiB/s wr, 37 op/s 2024-03-07T03:40:51.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:50 smithi145 bash[79878]: cluster 2024-03-07T03:40:49.477398+0000 mgr.y (mgr.34107) 869 : cluster 0 pgmap v1441: 105 pgs: 105 active+clean; 320 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.0 MiB/s wr, 34 op/s 2024-03-07T03:40:51.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:50 smithi145 bash[79878]: cluster 2024-03-07T03:40:49.725550+0000 mon.a (mon.0) 1692 : cluster 0 osdmap e948: 8 total, 8 up, 8 in 2024-03-07T03:40:51.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:50 smithi145 bash[82329]: cluster 2024-03-07T03:40:49.477398+0000 mgr.y (mgr.34107) 869 : cluster 0 pgmap v1441: 105 pgs: 105 active+clean; 320 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.0 MiB/s wr, 34 op/s 2024-03-07T03:40:51.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:50 smithi145 bash[82329]: cluster 2024-03-07T03:40:49.725550+0000 mon.a (mon.0) 1692 : cluster 0 osdmap e948: 8 total, 8 up, 8 in 2024-03-07T03:40:51.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:50 smithi189 bash[66969]: cluster 2024-03-07T03:40:49.477398+0000 mgr.y (mgr.34107) 869 : cluster 0 pgmap v1441: 105 pgs: 105 active+clean; 320 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.0 MiB/s wr, 34 op/s 2024-03-07T03:40:51.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:50 smithi189 bash[66969]: cluster 2024-03-07T03:40:49.725550+0000 mon.a (mon.0) 1692 : cluster 0 osdmap e948: 8 total, 8 up, 8 in 2024-03-07T03:40:52.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:40:52 smithi189 bash[59814]: ts=2024-03-07T03:40:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:40:53.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:52 smithi145 bash[79878]: cluster 2024-03-07T03:40:51.478098+0000 mgr.y (mgr.34107) 870 : cluster 0 pgmap v1443: 105 pgs: 105 active+clean; 341 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 489 KiB/s rd, 4.1 MiB/s wr, 31 op/s 2024-03-07T03:40:53.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:40:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:40:52] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.43.0" 2024-03-07T03:40:53.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:52 smithi145 bash[82329]: cluster 2024-03-07T03:40:51.478098+0000 mgr.y (mgr.34107) 870 : cluster 0 pgmap v1443: 105 pgs: 105 active+clean; 341 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 489 KiB/s rd, 4.1 MiB/s wr, 31 op/s 2024-03-07T03:40:53.079 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:52 smithi189 bash[66969]: cluster 2024-03-07T03:40:51.478098+0000 mgr.y (mgr.34107) 870 : cluster 0 pgmap v1443: 105 pgs: 105 active+clean; 341 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 489 KiB/s rd, 4.1 MiB/s wr, 31 op/s 2024-03-07T03:40:55.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:54 smithi145 bash[79878]: cluster 2024-03-07T03:40:53.478694+0000 mgr.y (mgr.34107) 871 : cluster 0 pgmap v1444: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 4.1 MiB/s wr, 41 op/s 2024-03-07T03:40:55.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:54 smithi145 bash[82329]: cluster 2024-03-07T03:40:53.478694+0000 mgr.y (mgr.34107) 871 : cluster 0 pgmap v1444: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 4.1 MiB/s wr, 41 op/s 2024-03-07T03:40:55.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:54 smithi189 bash[66969]: cluster 2024-03-07T03:40:53.478694+0000 mgr.y (mgr.34107) 871 : cluster 0 pgmap v1444: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 4.1 MiB/s wr, 41 op/s 2024-03-07T03:40:56.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:55 smithi145 bash[79878]: cluster 2024-03-07T03:40:54.740206+0000 mon.a (mon.0) 1693 : cluster 0 osdmap e949: 8 total, 8 up, 8 in 2024-03-07T03:40:56.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:55 smithi145 bash[82329]: cluster 2024-03-07T03:40:54.740206+0000 mon.a (mon.0) 1693 : cluster 0 osdmap e949: 8 total, 8 up, 8 in 2024-03-07T03:40:56.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:55 smithi189 bash[66969]: cluster 2024-03-07T03:40:54.740206+0000 mon.a (mon.0) 1693 : cluster 0 osdmap e949: 8 total, 8 up, 8 in 2024-03-07T03:40:57.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:56 smithi145 bash[79878]: cluster 2024-03-07T03:40:55.479263+0000 mgr.y (mgr.34107) 872 : cluster 0 pgmap v1446: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 5.1 MiB/s wr, 52 op/s 2024-03-07T03:40:57.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:56 smithi145 bash[82329]: cluster 2024-03-07T03:40:55.479263+0000 mgr.y (mgr.34107) 872 : cluster 0 pgmap v1446: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 5.1 MiB/s wr, 52 op/s 2024-03-07T03:40:57.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:56 smithi189 bash[66969]: cluster 2024-03-07T03:40:55.479263+0000 mgr.y (mgr.34107) 872 : cluster 0 pgmap v1446: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 5.1 MiB/s wr, 52 op/s 2024-03-07T03:40:57.268 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/5.ShrinkToOneAfterSnapshot (17715 ms) 2024-03-07T03:40:57.268 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/5.MinOverlap 2024-03-07T03:40:57.268 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:40:59.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:40:58 smithi145 bash[82329]: cluster 2024-03-07T03:40:57.480246+0000 mgr.y (mgr.34107) 873 : cluster 0 pgmap v1447: 105 pgs: 105 active+clean; 365 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 5.6 MiB/s wr, 72 op/s 2024-03-07T03:40:59.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:40:58 smithi145 bash[79878]: cluster 2024-03-07T03:40:57.480246+0000 mgr.y (mgr.34107) 873 : cluster 0 pgmap v1447: 105 pgs: 105 active+clean; 365 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 5.6 MiB/s wr, 72 op/s 2024-03-07T03:40:59.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:40:58 smithi189 bash[66969]: cluster 2024-03-07T03:40:57.480246+0000 mgr.y (mgr.34107) 873 : cluster 0 pgmap v1447: 105 pgs: 105 active+clean; 365 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 5.6 MiB/s wr, 72 op/s 2024-03-07T03:41:01.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:00 smithi145 bash[79878]: cluster 2024-03-07T03:40:59.481014+0000 mgr.y (mgr.34107) 874 : cluster 0 pgmap v1448: 105 pgs: 105 active+clean; 426 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 11 MiB/s wr, 76 op/s 2024-03-07T03:41:01.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:00 smithi145 bash[82329]: cluster 2024-03-07T03:40:59.481014+0000 mgr.y (mgr.34107) 874 : cluster 0 pgmap v1448: 105 pgs: 105 active+clean; 426 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 11 MiB/s wr, 76 op/s 2024-03-07T03:41:01.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:00 smithi189 bash[66969]: cluster 2024-03-07T03:40:59.481014+0000 mgr.y (mgr.34107) 874 : cluster 0 pgmap v1448: 105 pgs: 105 active+clean; 426 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 11 MiB/s wr, 76 op/s 2024-03-07T03:41:02.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:41:02 smithi189 bash[59814]: ts=2024-03-07T03:41:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:41:03.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:41:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:41:02] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.43.0" 2024-03-07T03:41:03.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:02 smithi145 bash[79878]: cluster 2024-03-07T03:41:01.481683+0000 mgr.y (mgr.34107) 875 : cluster 0 pgmap v1449: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 9.3 MiB/s wr, 65 op/s 2024-03-07T03:41:03.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:02 smithi145 bash[79878]: audit 2024-03-07T03:41:02.669985+0000 mon.a (mon.0) 1694 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:03.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:02 smithi145 bash[82329]: cluster 2024-03-07T03:41:01.481683+0000 mgr.y (mgr.34107) 875 : cluster 0 pgmap v1449: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 9.3 MiB/s wr, 65 op/s 2024-03-07T03:41:03.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:02 smithi145 bash[82329]: audit 2024-03-07T03:41:02.669985+0000 mon.a (mon.0) 1694 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:02 smithi189 bash[66969]: cluster 2024-03-07T03:41:01.481683+0000 mgr.y (mgr.34107) 875 : cluster 0 pgmap v1449: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 9.3 MiB/s wr, 65 op/s 2024-03-07T03:41:03.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:02 smithi189 bash[66969]: audit 2024-03-07T03:41:02.669985+0000 mon.a (mon.0) 1694 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:05.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:04 smithi145 bash[79878]: cluster 2024-03-07T03:41:03.482295+0000 mgr.y (mgr.34107) 876 : cluster 0 pgmap v1450: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 8.5 MiB/s wr, 44 op/s 2024-03-07T03:41:05.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:04 smithi145 bash[82329]: cluster 2024-03-07T03:41:03.482295+0000 mgr.y (mgr.34107) 876 : cluster 0 pgmap v1450: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 8.5 MiB/s wr, 44 op/s 2024-03-07T03:41:05.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:04 smithi189 bash[66969]: cluster 2024-03-07T03:41:03.482295+0000 mgr.y (mgr.34107) 876 : cluster 0 pgmap v1450: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 8.5 MiB/s wr, 44 op/s 2024-03-07T03:41:07.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:06 smithi145 bash[79878]: cluster 2024-03-07T03:41:05.483324+0000 mgr.y (mgr.34107) 877 : cluster 0 pgmap v1451: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 7.9 MiB/s wr, 39 op/s 2024-03-07T03:41:07.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:06 smithi145 bash[79878]: cluster 2024-03-07T03:41:05.773454+0000 mon.a (mon.0) 1695 : cluster 0 osdmap e950: 8 total, 8 up, 8 in 2024-03-07T03:41:07.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:06 smithi145 bash[82329]: cluster 2024-03-07T03:41:05.483324+0000 mgr.y (mgr.34107) 877 : cluster 0 pgmap v1451: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 7.9 MiB/s wr, 39 op/s 2024-03-07T03:41:07.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:06 smithi145 bash[82329]: cluster 2024-03-07T03:41:05.773454+0000 mon.a (mon.0) 1695 : cluster 0 osdmap e950: 8 total, 8 up, 8 in 2024-03-07T03:41:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:06 smithi189 bash[66969]: cluster 2024-03-07T03:41:05.483324+0000 mgr.y (mgr.34107) 877 : cluster 0 pgmap v1451: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 7.9 MiB/s wr, 39 op/s 2024-03-07T03:41:07.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:06 smithi189 bash[66969]: cluster 2024-03-07T03:41:05.773454+0000 mon.a (mon.0) 1695 : cluster 0 osdmap e950: 8 total, 8 up, 8 in 2024-03-07T03:41:09.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:08 smithi145 bash[79878]: cluster 2024-03-07T03:41:07.483969+0000 mgr.y (mgr.34107) 878 : cluster 0 pgmap v1453: 105 pgs: 105 active+clean; 446 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 8.1 MiB/s wr, 19 op/s 2024-03-07T03:41:09.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:08 smithi145 bash[82329]: cluster 2024-03-07T03:41:07.483969+0000 mgr.y (mgr.34107) 878 : cluster 0 pgmap v1453: 105 pgs: 105 active+clean; 446 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 8.1 MiB/s wr, 19 op/s 2024-03-07T03:41:09.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:08 smithi189 bash[66969]: cluster 2024-03-07T03:41:07.483969+0000 mgr.y (mgr.34107) 878 : cluster 0 pgmap v1453: 105 pgs: 105 active+clean; 446 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 8.1 MiB/s wr, 19 op/s 2024-03-07T03:41:11.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:10 smithi145 bash[79878]: cluster 2024-03-07T03:41:09.484587+0000 mgr.y (mgr.34107) 879 : cluster 0 pgmap v1454: 105 pgs: 105 active+clean; 467 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.1 MiB/s wr, 32 op/s 2024-03-07T03:41:11.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:10 smithi145 bash[82329]: cluster 2024-03-07T03:41:09.484587+0000 mgr.y (mgr.34107) 879 : cluster 0 pgmap v1454: 105 pgs: 105 active+clean; 467 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.1 MiB/s wr, 32 op/s 2024-03-07T03:41:11.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:10 smithi189 bash[66969]: cluster 2024-03-07T03:41:09.484587+0000 mgr.y (mgr.34107) 879 : cluster 0 pgmap v1454: 105 pgs: 105 active+clean; 467 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.1 MiB/s wr, 32 op/s 2024-03-07T03:41:11.724 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/5.MinOverlap (14455 ms) 2024-03-07T03:41:11.724 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/5.ZeroOverlap 2024-03-07T03:41:11.724 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:41:12.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:41:12 smithi189 bash[59814]: ts=2024-03-07T03:41:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:41:13.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:12 smithi145 bash[79878]: cluster 2024-03-07T03:41:11.485233+0000 mgr.y (mgr.34107) 880 : cluster 0 pgmap v1455: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.7 MiB/s wr, 58 op/s 2024-03-07T03:41:13.060 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:41:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:41:12] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.43.0" 2024-03-07T03:41:13.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:12 smithi145 bash[82329]: cluster 2024-03-07T03:41:11.485233+0000 mgr.y (mgr.34107) 880 : cluster 0 pgmap v1455: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.7 MiB/s wr, 58 op/s 2024-03-07T03:41:13.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:12 smithi189 bash[66969]: cluster 2024-03-07T03:41:11.485233+0000 mgr.y (mgr.34107) 880 : cluster 0 pgmap v1455: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.7 MiB/s wr, 58 op/s 2024-03-07T03:41:15.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:14 smithi145 bash[79878]: cluster 2024-03-07T03:41:13.485904+0000 mgr.y (mgr.34107) 881 : cluster 0 pgmap v1456: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.7 MiB/s wr, 57 op/s 2024-03-07T03:41:15.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:14 smithi145 bash[82329]: cluster 2024-03-07T03:41:13.485904+0000 mgr.y (mgr.34107) 881 : cluster 0 pgmap v1456: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.7 MiB/s wr, 57 op/s 2024-03-07T03:41:15.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:14 smithi189 bash[66969]: cluster 2024-03-07T03:41:13.485904+0000 mgr.y (mgr.34107) 881 : cluster 0 pgmap v1456: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.7 MiB/s wr, 57 op/s 2024-03-07T03:41:17.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:16 smithi145 bash[79878]: cluster 2024-03-07T03:41:15.486615+0000 mgr.y (mgr.34107) 882 : cluster 0 pgmap v1457: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 62 op/s 2024-03-07T03:41:17.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:16 smithi145 bash[82329]: cluster 2024-03-07T03:41:15.486615+0000 mgr.y (mgr.34107) 882 : cluster 0 pgmap v1457: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 62 op/s 2024-03-07T03:41:17.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:16 smithi189 bash[66969]: cluster 2024-03-07T03:41:15.486615+0000 mgr.y (mgr.34107) 882 : cluster 0 pgmap v1457: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 62 op/s 2024-03-07T03:41:18.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:17 smithi145 bash[79878]: audit 2024-03-07T03:41:17.670311+0000 mon.a (mon.0) 1696 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:18.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:17 smithi145 bash[82329]: audit 2024-03-07T03:41:17.670311+0000 mon.a (mon.0) 1696 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:18.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:17 smithi189 bash[66969]: audit 2024-03-07T03:41:17.670311+0000 mon.a (mon.0) 1696 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:19.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:18 smithi145 bash[82329]: cluster 2024-03-07T03:41:17.487302+0000 mgr.y (mgr.34107) 883 : cluster 0 pgmap v1458: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.2 MiB/s wr, 54 op/s 2024-03-07T03:41:19.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:18 smithi145 bash[79878]: cluster 2024-03-07T03:41:17.487302+0000 mgr.y (mgr.34107) 883 : cluster 0 pgmap v1458: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.2 MiB/s wr, 54 op/s 2024-03-07T03:41:19.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:18 smithi189 bash[66969]: cluster 2024-03-07T03:41:17.487302+0000 mgr.y (mgr.34107) 883 : cluster 0 pgmap v1458: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.2 MiB/s wr, 54 op/s 2024-03-07T03:41:21.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:20 smithi145 bash[79878]: cluster 2024-03-07T03:41:19.488252+0000 mgr.y (mgr.34107) 884 : cluster 0 pgmap v1459: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.4 MiB/s wr, 51 op/s 2024-03-07T03:41:21.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:20 smithi145 bash[79878]: cluster 2024-03-07T03:41:19.814007+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e951: 8 total, 8 up, 8 in 2024-03-07T03:41:21.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:20 smithi145 bash[82329]: cluster 2024-03-07T03:41:19.488252+0000 mgr.y (mgr.34107) 884 : cluster 0 pgmap v1459: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.4 MiB/s wr, 51 op/s 2024-03-07T03:41:21.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:20 smithi145 bash[82329]: cluster 2024-03-07T03:41:19.814007+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e951: 8 total, 8 up, 8 in 2024-03-07T03:41:21.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:20 smithi189 bash[66969]: cluster 2024-03-07T03:41:19.488252+0000 mgr.y (mgr.34107) 884 : cluster 0 pgmap v1459: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.4 MiB/s wr, 51 op/s 2024-03-07T03:41:21.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:20 smithi189 bash[66969]: cluster 2024-03-07T03:41:19.814007+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e951: 8 total, 8 up, 8 in 2024-03-07T03:41:22.580 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:41:22 smithi189 bash[59814]: ts=2024-03-07T03:41:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:41:23.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:22 smithi145 bash[79878]: cluster 2024-03-07T03:41:21.488971+0000 mgr.y (mgr.34107) 885 : cluster 0 pgmap v1461: 105 pgs: 105 active+clean; 512 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-03-07T03:41:23.061 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:41:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:41:22] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.43.0" 2024-03-07T03:41:23.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:22 smithi145 bash[82329]: cluster 2024-03-07T03:41:21.488971+0000 mgr.y (mgr.34107) 885 : cluster 0 pgmap v1461: 105 pgs: 105 active+clean; 512 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-03-07T03:41:23.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:22 smithi189 bash[66969]: cluster 2024-03-07T03:41:21.488971+0000 mgr.y (mgr.34107) 885 : cluster 0 pgmap v1461: 105 pgs: 105 active+clean; 512 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-03-07T03:41:25.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:24 smithi145 bash[79878]: cluster 2024-03-07T03:41:23.489652+0000 mgr.y (mgr.34107) 886 : cluster 0 pgmap v1462: 105 pgs: 105 active+clean; 516 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-03-07T03:41:25.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:24 smithi145 bash[82329]: cluster 2024-03-07T03:41:23.489652+0000 mgr.y (mgr.34107) 886 : cluster 0 pgmap v1462: 105 pgs: 105 active+clean; 516 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-03-07T03:41:25.080 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:24 smithi189 bash[66969]: cluster 2024-03-07T03:41:23.489652+0000 mgr.y (mgr.34107) 886 : cluster 0 pgmap v1462: 105 pgs: 105 active+clean; 516 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.9 MiB/s wr, 27 op/s 2024-03-07T03:41:25.681 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/5.ZeroOverlap (13957 ms) 2024-03-07T03:41:25.850 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/5 (104102 ms total) 2024-03-07T03:41:25.850 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:41:25.850 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/6, where TypeParam = LUKS2UnderLUKS2 2024-03-07T03:41:25.910 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/6.Simple 2024-03-07T03:41:27.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:26 smithi145 bash[79878]: cluster 2024-03-07T03:41:25.490812+0000 mgr.y (mgr.34107) 887 : cluster 0 pgmap v1463: 105 pgs: 105 active+clean; 540 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 909 KiB/s rd, 4.5 MiB/s wr, 45 op/s 2024-03-07T03:41:27.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:26 smithi145 bash[79878]: cluster 2024-03-07T03:41:25.835343+0000 mon.a (mon.0) 1698 : cluster 0 osdmap e952: 8 total, 8 up, 8 in 2024-03-07T03:41:27.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:26 smithi145 bash[82329]: cluster 2024-03-07T03:41:25.490812+0000 mgr.y (mgr.34107) 887 : cluster 0 pgmap v1463: 105 pgs: 105 active+clean; 540 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 909 KiB/s rd, 4.5 MiB/s wr, 45 op/s 2024-03-07T03:41:27.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:26 smithi145 bash[82329]: cluster 2024-03-07T03:41:25.835343+0000 mon.a (mon.0) 1698 : cluster 0 osdmap e952: 8 total, 8 up, 8 in 2024-03-07T03:41:27.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:26 smithi189 bash[66969]: cluster 2024-03-07T03:41:25.490812+0000 mgr.y (mgr.34107) 887 : cluster 0 pgmap v1463: 105 pgs: 105 active+clean; 540 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 909 KiB/s rd, 4.5 MiB/s wr, 45 op/s 2024-03-07T03:41:27.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:26 smithi189 bash[66969]: cluster 2024-03-07T03:41:25.835343+0000 mon.a (mon.0) 1698 : cluster 0 osdmap e952: 8 total, 8 up, 8 in 2024-03-07T03:41:27.841 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:41:28.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:27 smithi145 bash[79878]: cluster 2024-03-07T03:41:26.839691+0000 mon.a (mon.0) 1699 : cluster 0 osdmap e953: 8 total, 8 up, 8 in 2024-03-07T03:41:28.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:27 smithi145 bash[79878]: audit 2024-03-07T03:41:26.841907+0000 mon.a (mon.0) 1700 : audit 1 from='client.? 172.21.15.145:0/1692385842' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-42","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:41:28.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:27 smithi145 bash[79878]: audit 2024-03-07T03:41:27.835436+0000 mon.a (mon.0) 1701 : audit 1 from='client.? 172.21.15.145:0/1692385842' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-42","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:41:28.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:27 smithi145 bash[82329]: cluster 2024-03-07T03:41:26.839691+0000 mon.a (mon.0) 1699 : cluster 0 osdmap e953: 8 total, 8 up, 8 in 2024-03-07T03:41:28.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:27 smithi145 bash[82329]: audit 2024-03-07T03:41:26.841907+0000 mon.a (mon.0) 1700 : audit 1 from='client.? 172.21.15.145:0/1692385842' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-42","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:41:28.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:27 smithi145 bash[82329]: audit 2024-03-07T03:41:27.835436+0000 mon.a (mon.0) 1701 : audit 1 from='client.? 172.21.15.145:0/1692385842' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-42","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:41:28.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:27 smithi189 bash[66969]: cluster 2024-03-07T03:41:26.839691+0000 mon.a (mon.0) 1699 : cluster 0 osdmap e953: 8 total, 8 up, 8 in 2024-03-07T03:41:28.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:27 smithi189 bash[66969]: audit 2024-03-07T03:41:26.841907+0000 mon.a (mon.0) 1700 : audit 1 from='client.? 172.21.15.145:0/1692385842' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-42","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:41:28.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:27 smithi189 bash[66969]: audit 2024-03-07T03:41:27.835436+0000 mon.a (mon.0) 1701 : audit 1 from='client.? 172.21.15.145:0/1692385842' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-42","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:41:29.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:28 smithi145 bash[79878]: cluster 2024-03-07T03:41:27.491509+0000 mgr.y (mgr.34107) 888 : cluster 0 pgmap v1466: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:41:29.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:28 smithi145 bash[79878]: cluster 2024-03-07T03:41:27.844956+0000 mon.a (mon.0) 1702 : cluster 0 osdmap e954: 8 total, 8 up, 8 in 2024-03-07T03:41:29.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:28 smithi145 bash[79878]: cluster 2024-03-07T03:41:27.846836+0000 mon.a (mon.0) 1703 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:41:29.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:28 smithi145 bash[79878]: cluster 2024-03-07T03:41:28.866280+0000 mon.a (mon.0) 1704 : cluster 0 osdmap e955: 8 total, 8 up, 8 in 2024-03-07T03:41:29.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:28 smithi145 bash[82329]: cluster 2024-03-07T03:41:27.491509+0000 mgr.y (mgr.34107) 888 : cluster 0 pgmap v1466: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:41:29.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:28 smithi145 bash[82329]: cluster 2024-03-07T03:41:27.844956+0000 mon.a (mon.0) 1702 : cluster 0 osdmap e954: 8 total, 8 up, 8 in 2024-03-07T03:41:29.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:28 smithi145 bash[82329]: cluster 2024-03-07T03:41:27.846836+0000 mon.a (mon.0) 1703 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:41:29.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:28 smithi145 bash[82329]: cluster 2024-03-07T03:41:28.866280+0000 mon.a (mon.0) 1704 : cluster 0 osdmap e955: 8 total, 8 up, 8 in 2024-03-07T03:41:29.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:28 smithi189 bash[66969]: cluster 2024-03-07T03:41:27.491509+0000 mgr.y (mgr.34107) 888 : cluster 0 pgmap v1466: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:41:29.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:28 smithi189 bash[66969]: cluster 2024-03-07T03:41:27.844956+0000 mon.a (mon.0) 1702 : cluster 0 osdmap e954: 8 total, 8 up, 8 in 2024-03-07T03:41:29.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:28 smithi189 bash[66969]: cluster 2024-03-07T03:41:27.846836+0000 mon.a (mon.0) 1703 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:41:29.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:28 smithi189 bash[66969]: cluster 2024-03-07T03:41:28.866280+0000 mon.a (mon.0) 1704 : cluster 0 osdmap e955: 8 total, 8 up, 8 in 2024-03-07T03:41:30.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:29 smithi145 bash[79878]: cluster 2024-03-07T03:41:29.874834+0000 mon.a (mon.0) 1705 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:41:30.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:29 smithi145 bash[79878]: cluster 2024-03-07T03:41:29.889151+0000 mon.a (mon.0) 1706 : cluster 0 osdmap e956: 8 total, 8 up, 8 in 2024-03-07T03:41:30.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:29 smithi145 bash[82329]: cluster 2024-03-07T03:41:29.874834+0000 mon.a (mon.0) 1705 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:41:30.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:29 smithi145 bash[82329]: cluster 2024-03-07T03:41:29.889151+0000 mon.a (mon.0) 1706 : cluster 0 osdmap e956: 8 total, 8 up, 8 in 2024-03-07T03:41:30.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:29 smithi189 bash[66969]: cluster 2024-03-07T03:41:29.874834+0000 mon.a (mon.0) 1705 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:41:30.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:29 smithi189 bash[66969]: cluster 2024-03-07T03:41:29.889151+0000 mon.a (mon.0) 1706 : cluster 0 osdmap e956: 8 total, 8 up, 8 in 2024-03-07T03:41:31.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:30 smithi145 bash[79878]: cluster 2024-03-07T03:41:29.492490+0000 mgr.y (mgr.34107) 889 : cluster 0 pgmap v1469: 105 pgs: 20 unknown, 85 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:41:31.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:30 smithi145 bash[82329]: cluster 2024-03-07T03:41:29.492490+0000 mgr.y (mgr.34107) 889 : cluster 0 pgmap v1469: 105 pgs: 20 unknown, 85 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:41:31.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:30 smithi189 bash[66969]: cluster 2024-03-07T03:41:29.492490+0000 mgr.y (mgr.34107) 889 : cluster 0 pgmap v1469: 105 pgs: 20 unknown, 85 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:41:32.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:41:32 smithi189 bash[59814]: ts=2024-03-07T03:41:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:41:33.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:32 smithi145 bash[79878]: cluster 2024-03-07T03:41:31.493259+0000 mgr.y (mgr.34107) 890 : cluster 0 pgmap v1471: 105 pgs: 10 unknown, 95 active+clean; 582 KiB data, 932 MiB used, 714 GiB / 715 GiB avail; 5.2 KiB/s rd, 879 B/s wr, 7 op/s 2024-03-07T03:41:33.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:32 smithi145 bash[79878]: audit 2024-03-07T03:41:32.670786+0000 mon.a (mon.0) 1707 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:33.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:32 smithi145 bash[82329]: cluster 2024-03-07T03:41:31.493259+0000 mgr.y (mgr.34107) 890 : cluster 0 pgmap v1471: 105 pgs: 10 unknown, 95 active+clean; 582 KiB data, 932 MiB used, 714 GiB / 715 GiB avail; 5.2 KiB/s rd, 879 B/s wr, 7 op/s 2024-03-07T03:41:33.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:32 smithi145 bash[82329]: audit 2024-03-07T03:41:32.670786+0000 mon.a (mon.0) 1707 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:33.061 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:41:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:41:32] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2024-03-07T03:41:33.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:32 smithi189 bash[66969]: cluster 2024-03-07T03:41:31.493259+0000 mgr.y (mgr.34107) 890 : cluster 0 pgmap v1471: 105 pgs: 10 unknown, 95 active+clean; 582 KiB data, 932 MiB used, 714 GiB / 715 GiB avail; 5.2 KiB/s rd, 879 B/s wr, 7 op/s 2024-03-07T03:41:33.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:32 smithi189 bash[66969]: audit 2024-03-07T03:41:32.670786+0000 mon.a (mon.0) 1707 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:35.310 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:34 smithi145 bash[79878]: cluster 2024-03-07T03:41:33.493903+0000 mgr.y (mgr.34107) 891 : cluster 0 pgmap v1472: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 682 B/s wr, 5 op/s 2024-03-07T03:41:35.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:34 smithi145 bash[82329]: cluster 2024-03-07T03:41:33.493903+0000 mgr.y (mgr.34107) 891 : cluster 0 pgmap v1472: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 682 B/s wr, 5 op/s 2024-03-07T03:41:35.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:34 smithi189 bash[66969]: cluster 2024-03-07T03:41:33.493903+0000 mgr.y (mgr.34107) 891 : cluster 0 pgmap v1472: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 682 B/s wr, 5 op/s 2024-03-07T03:41:37.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:36 smithi145 bash[79878]: cluster 2024-03-07T03:41:35.494687+0000 mgr.y (mgr.34107) 892 : cluster 0 pgmap v1473: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-03-07T03:41:37.311 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:36 smithi145 bash[82329]: cluster 2024-03-07T03:41:35.494687+0000 mgr.y (mgr.34107) 892 : cluster 0 pgmap v1473: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-03-07T03:41:37.330 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:36 smithi189 bash[66969]: cluster 2024-03-07T03:41:35.494687+0000 mgr.y (mgr.34107) 892 : cluster 0 pgmap v1473: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-03-07T03:41:39.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:39 smithi145 bash[79878]: cluster 2024-03-07T03:41:37.495383+0000 mgr.y (mgr.34107) 893 : cluster 0 pgmap v1474: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 948 B/s wr, 6 op/s 2024-03-07T03:41:39.311 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:39 smithi145 bash[79878]: cluster 2024-03-07T03:41:38.023374+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e957: 8 total, 8 up, 8 in 2024-03-07T03:41:39.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:39 smithi145 bash[82329]: cluster 2024-03-07T03:41:37.495383+0000 mgr.y (mgr.34107) 893 : cluster 0 pgmap v1474: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 948 B/s wr, 6 op/s 2024-03-07T03:41:39.312 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:39 smithi145 bash[82329]: cluster 2024-03-07T03:41:38.023374+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e957: 8 total, 8 up, 8 in 2024-03-07T03:41:39.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:39 smithi189 bash[66969]: cluster 2024-03-07T03:41:37.495383+0000 mgr.y (mgr.34107) 893 : cluster 0 pgmap v1474: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 948 B/s wr, 6 op/s 2024-03-07T03:41:39.331 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:39 smithi189 bash[66969]: cluster 2024-03-07T03:41:38.023374+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e957: 8 total, 8 up, 8 in 2024-03-07T03:41:41.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:40 smithi145 bash[79878]: cluster 2024-03-07T03:41:39.496270+0000 mgr.y (mgr.34107) 894 : cluster 0 pgmap v1476: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 852 B/s wr, 5 op/s 2024-03-07T03:41:41.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:40 smithi145 bash[79878]: audit 2024-03-07T03:41:39.663208+0000 mon.a (mon.0) 1709 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:41.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:40 smithi145 bash[79878]: cluster 2024-03-07T03:41:40.039318+0000 mon.a (mon.0) 1710 : cluster 0 osdmap e958: 8 total, 8 up, 8 in 2024-03-07T03:41:41.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:40 smithi145 bash[82329]: cluster 2024-03-07T03:41:39.496270+0000 mgr.y (mgr.34107) 894 : cluster 0 pgmap v1476: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 852 B/s wr, 5 op/s 2024-03-07T03:41:41.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:40 smithi145 bash[82329]: audit 2024-03-07T03:41:39.663208+0000 mon.a (mon.0) 1709 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:41.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:40 smithi145 bash[82329]: cluster 2024-03-07T03:41:40.039318+0000 mon.a (mon.0) 1710 : cluster 0 osdmap e958: 8 total, 8 up, 8 in 2024-03-07T03:41:41.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:40 smithi189 bash[66969]: cluster 2024-03-07T03:41:39.496270+0000 mgr.y (mgr.34107) 894 : cluster 0 pgmap v1476: 105 pgs: 105 active+clean; 582 KiB data, 586 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 852 B/s wr, 5 op/s 2024-03-07T03:41:41.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:40 smithi189 bash[66969]: audit 2024-03-07T03:41:39.663208+0000 mon.a (mon.0) 1709 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:41.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:40 smithi189 bash[66969]: cluster 2024-03-07T03:41:40.039318+0000 mon.a (mon.0) 1710 : cluster 0 osdmap e958: 8 total, 8 up, 8 in 2024-03-07T03:41:42.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:41 smithi145 bash[79878]: audit 2024-03-07T03:41:41.168562+0000 mon.a (mon.0) 1711 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:41:42.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:41 smithi145 bash[82329]: audit 2024-03-07T03:41:41.168562+0000 mon.a (mon.0) 1711 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:41:42.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:41 smithi189 bash[66969]: audit 2024-03-07T03:41:41.168562+0000 mon.a (mon.0) 1711 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:41:42.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:41:42 smithi189 bash[59814]: ts=2024-03-07T03:41:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:41:43.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:42 smithi145 bash[79878]: cluster 2024-03-07T03:41:41.496915+0000 mgr.y (mgr.34107) 895 : cluster 0 pgmap v1478: 105 pgs: 105 active+clean; 18 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 2.1 MiB/s wr, 24 op/s 2024-03-07T03:41:43.061 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:41:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:41:42] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2024-03-07T03:41:43.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:42 smithi145 bash[82329]: cluster 2024-03-07T03:41:41.496915+0000 mgr.y (mgr.34107) 895 : cluster 0 pgmap v1478: 105 pgs: 105 active+clean; 18 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 2.1 MiB/s wr, 24 op/s 2024-03-07T03:41:43.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:42 smithi189 bash[66969]: cluster 2024-03-07T03:41:41.496915+0000 mgr.y (mgr.34107) 895 : cluster 0 pgmap v1478: 105 pgs: 105 active+clean; 18 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 2.1 MiB/s wr, 24 op/s 2024-03-07T03:41:45.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:44 smithi145 bash[79878]: cluster 2024-03-07T03:41:43.497568+0000 mgr.y (mgr.34107) 896 : cluster 0 pgmap v1479: 105 pgs: 105 active+clean; 30 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 3.6 MiB/s wr, 22 op/s 2024-03-07T03:41:45.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:44 smithi145 bash[82329]: cluster 2024-03-07T03:41:43.497568+0000 mgr.y (mgr.34107) 896 : cluster 0 pgmap v1479: 105 pgs: 105 active+clean; 30 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 3.6 MiB/s wr, 22 op/s 2024-03-07T03:41:45.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:44 smithi189 bash[66969]: cluster 2024-03-07T03:41:43.497568+0000 mgr.y (mgr.34107) 896 : cluster 0 pgmap v1479: 105 pgs: 105 active+clean; 30 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 3.6 MiB/s wr, 22 op/s 2024-03-07T03:41:47.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:46 smithi145 bash[79878]: cluster 2024-03-07T03:41:45.498291+0000 mgr.y (mgr.34107) 897 : cluster 0 pgmap v1480: 105 pgs: 105 active+clean; 42 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 5.1 MiB/s wr, 44 op/s 2024-03-07T03:41:47.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:46 smithi145 bash[82329]: cluster 2024-03-07T03:41:45.498291+0000 mgr.y (mgr.34107) 897 : cluster 0 pgmap v1480: 105 pgs: 105 active+clean; 42 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 5.1 MiB/s wr, 44 op/s 2024-03-07T03:41:47.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:46 smithi189 bash[66969]: cluster 2024-03-07T03:41:45.498291+0000 mgr.y (mgr.34107) 897 : cluster 0 pgmap v1480: 105 pgs: 105 active+clean; 42 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 5.1 MiB/s wr, 44 op/s 2024-03-07T03:41:48.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[79878]: audit 2024-03-07T03:41:47.163011+0000 mon.a (mon.0) 1712 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[79878]: audit 2024-03-07T03:41:47.172798+0000 mon.a (mon.0) 1713 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.561 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[79878]: audit 2024-03-07T03:41:47.215294+0000 mon.a (mon.0) 1714 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[79878]: audit 2024-03-07T03:41:47.225918+0000 mon.a (mon.0) 1715 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[79878]: cluster 2024-03-07T03:41:47.498987+0000 mgr.y (mgr.34107) 898 : cluster 0 pgmap v1481: 105 pgs: 105 active+clean; 42 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 4.3 MiB/s wr, 37 op/s 2024-03-07T03:41:48.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[79878]: audit 2024-03-07T03:41:47.671559+0000 mon.a (mon.0) 1716 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:48.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[79878]: audit 2024-03-07T03:41:47.913757+0000 mon.a (mon.0) 1717 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:41:48.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[79878]: audit 2024-03-07T03:41:47.915469+0000 mon.a (mon.0) 1718 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:41:48.562 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[79878]: audit 2024-03-07T03:41:47.923760+0000 mon.a (mon.0) 1719 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.562 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[82329]: audit 2024-03-07T03:41:47.163011+0000 mon.a (mon.0) 1712 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[82329]: audit 2024-03-07T03:41:47.172798+0000 mon.a (mon.0) 1713 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[82329]: audit 2024-03-07T03:41:47.215294+0000 mon.a (mon.0) 1714 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[82329]: audit 2024-03-07T03:41:47.225918+0000 mon.a (mon.0) 1715 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[82329]: cluster 2024-03-07T03:41:47.498987+0000 mgr.y (mgr.34107) 898 : cluster 0 pgmap v1481: 105 pgs: 105 active+clean; 42 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 4.3 MiB/s wr, 37 op/s 2024-03-07T03:41:48.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[82329]: audit 2024-03-07T03:41:47.671559+0000 mon.a (mon.0) 1716 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:48.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[82329]: audit 2024-03-07T03:41:47.913757+0000 mon.a (mon.0) 1717 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:41:48.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[82329]: audit 2024-03-07T03:41:47.915469+0000 mon.a (mon.0) 1718 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:41:48.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:48 smithi145 bash[82329]: audit 2024-03-07T03:41:47.923760+0000 mon.a (mon.0) 1719 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:48 smithi189 bash[66969]: audit 2024-03-07T03:41:47.163011+0000 mon.a (mon.0) 1712 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:48 smithi189 bash[66969]: audit 2024-03-07T03:41:47.172798+0000 mon.a (mon.0) 1713 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:48 smithi189 bash[66969]: audit 2024-03-07T03:41:47.215294+0000 mon.a (mon.0) 1714 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:48 smithi189 bash[66969]: audit 2024-03-07T03:41:47.225918+0000 mon.a (mon.0) 1715 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:48.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:48 smithi189 bash[66969]: cluster 2024-03-07T03:41:47.498987+0000 mgr.y (mgr.34107) 898 : cluster 0 pgmap v1481: 105 pgs: 105 active+clean; 42 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 4.3 MiB/s wr, 37 op/s 2024-03-07T03:41:48.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:48 smithi189 bash[66969]: audit 2024-03-07T03:41:47.671559+0000 mon.a (mon.0) 1716 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:48.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:48 smithi189 bash[66969]: audit 2024-03-07T03:41:47.913757+0000 mon.a (mon.0) 1717 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:41:48.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:48 smithi189 bash[66969]: audit 2024-03-07T03:41:47.915469+0000 mon.a (mon.0) 1718 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:41:48.582 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:48 smithi189 bash[66969]: audit 2024-03-07T03:41:47.923760+0000 mon.a (mon.0) 1719 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:41:50.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:50 smithi145 bash[79878]: cluster 2024-03-07T03:41:49.499643+0000 mgr.y (mgr.34107) 899 : cluster 0 pgmap v1482: 105 pgs: 105 active+clean; 50 MiB data, 742 MiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 4.9 MiB/s wr, 43 op/s 2024-03-07T03:41:50.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:50 smithi145 bash[82329]: cluster 2024-03-07T03:41:49.499643+0000 mgr.y (mgr.34107) 899 : cluster 0 pgmap v1482: 105 pgs: 105 active+clean; 50 MiB data, 742 MiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 4.9 MiB/s wr, 43 op/s 2024-03-07T03:41:50.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:50 smithi189 bash[66969]: cluster 2024-03-07T03:41:49.499643+0000 mgr.y (mgr.34107) 899 : cluster 0 pgmap v1482: 105 pgs: 105 active+clean; 50 MiB data, 742 MiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 4.9 MiB/s wr, 43 op/s 2024-03-07T03:41:52.581 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:52 smithi189 bash[66969]: cluster 2024-03-07T03:41:51.500281+0000 mgr.y (mgr.34107) 900 : cluster 0 pgmap v1483: 105 pgs: 105 active+clean; 59 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 4.7 MiB/s wr, 41 op/s 2024-03-07T03:41:52.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:41:52 smithi189 bash[59814]: ts=2024-03-07T03:41:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:41:52.700 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/6.Simple (26790 ms) 2024-03-07T03:41:52.700 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/6.Grow 2024-03-07T03:41:52.700 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:41:52.811 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:41:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:41:52] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.43.0" 2024-03-07T03:41:52.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:52 smithi145 bash[79878]: cluster 2024-03-07T03:41:51.500281+0000 mgr.y (mgr.34107) 900 : cluster 0 pgmap v1483: 105 pgs: 105 active+clean; 59 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 4.7 MiB/s wr, 41 op/s 2024-03-07T03:41:52.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:52 smithi145 bash[82329]: cluster 2024-03-07T03:41:51.500281+0000 mgr.y (mgr.34107) 900 : cluster 0 pgmap v1483: 105 pgs: 105 active+clean; 59 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 4.7 MiB/s wr, 41 op/s 2024-03-07T03:41:54.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:54 smithi145 bash[79878]: cluster 2024-03-07T03:41:53.501095+0000 mgr.y (mgr.34107) 901 : cluster 0 pgmap v1484: 105 pgs: 105 active+clean; 75 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:41:54.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:54 smithi145 bash[82329]: cluster 2024-03-07T03:41:53.501095+0000 mgr.y (mgr.34107) 901 : cluster 0 pgmap v1484: 105 pgs: 105 active+clean; 75 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:41:54.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:54 smithi189 bash[66969]: cluster 2024-03-07T03:41:53.501095+0000 mgr.y (mgr.34107) 901 : cluster 0 pgmap v1484: 105 pgs: 105 active+clean; 75 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:41:56.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:56 smithi145 bash[82329]: cluster 2024-03-07T03:41:55.501843+0000 mgr.y (mgr.34107) 902 : cluster 0 pgmap v1485: 105 pgs: 105 active+clean; 83 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 5.1 MiB/s wr, 39 op/s 2024-03-07T03:41:56.812 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:56 smithi145 bash[79878]: cluster 2024-03-07T03:41:55.501843+0000 mgr.y (mgr.34107) 902 : cluster 0 pgmap v1485: 105 pgs: 105 active+clean; 83 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 5.1 MiB/s wr, 39 op/s 2024-03-07T03:41:56.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:56 smithi189 bash[66969]: cluster 2024-03-07T03:41:55.501843+0000 mgr.y (mgr.34107) 902 : cluster 0 pgmap v1485: 105 pgs: 105 active+clean; 83 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 5.1 MiB/s wr, 39 op/s 2024-03-07T03:41:58.811 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:41:58 smithi145 bash[79878]: cluster 2024-03-07T03:41:57.502574+0000 mgr.y (mgr.34107) 903 : cluster 0 pgmap v1486: 105 pgs: 105 active+clean; 83 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 28 op/s 2024-03-07T03:41:58.811 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:41:58 smithi145 bash[82329]: cluster 2024-03-07T03:41:57.502574+0000 mgr.y (mgr.34107) 903 : cluster 0 pgmap v1486: 105 pgs: 105 active+clean; 83 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 28 op/s 2024-03-07T03:41:58.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:41:58 smithi189 bash[66969]: cluster 2024-03-07T03:41:57.502574+0000 mgr.y (mgr.34107) 903 : cluster 0 pgmap v1486: 105 pgs: 105 active+clean; 83 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 28 op/s 2024-03-07T03:42:00.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:00 smithi189 bash[66969]: cluster 2024-03-07T03:41:59.503231+0000 mgr.y (mgr.34107) 904 : cluster 0 pgmap v1487: 105 pgs: 105 active+clean; 83 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 28 op/s 2024-03-07T03:42:01.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:00 smithi145 bash[79878]: cluster 2024-03-07T03:41:59.503231+0000 mgr.y (mgr.34107) 904 : cluster 0 pgmap v1487: 105 pgs: 105 active+clean; 83 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 28 op/s 2024-03-07T03:42:01.061 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:00 smithi145 bash[82329]: cluster 2024-03-07T03:41:59.503231+0000 mgr.y (mgr.34107) 904 : cluster 0 pgmap v1487: 105 pgs: 105 active+clean; 83 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 28 op/s 2024-03-07T03:42:01.831 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:01 smithi189 bash[66969]: cluster 2024-03-07T03:42:00.588371+0000 mon.a (mon.0) 1720 : cluster 0 osdmap e959: 8 total, 8 up, 8 in 2024-03-07T03:42:02.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:01 smithi145 bash[79878]: cluster 2024-03-07T03:42:00.588371+0000 mon.a (mon.0) 1720 : cluster 0 osdmap e959: 8 total, 8 up, 8 in 2024-03-07T03:42:02.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:01 smithi145 bash[82329]: cluster 2024-03-07T03:42:00.588371+0000 mon.a (mon.0) 1720 : cluster 0 osdmap e959: 8 total, 8 up, 8 in 2024-03-07T03:42:02.581 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:42:02 smithi189 bash[59814]: ts=2024-03-07T03:42:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:42:03.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:02 smithi145 bash[79878]: cluster 2024-03-07T03:42:01.504042+0000 mgr.y (mgr.34107) 905 : cluster 0 pgmap v1489: 105 pgs: 105 active+clean; 99 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 5.6 MiB/s wr, 36 op/s 2024-03-07T03:42:03.062 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:42:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:42:02] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.43.0" 2024-03-07T03:42:03.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:02 smithi145 bash[82329]: cluster 2024-03-07T03:42:01.504042+0000 mgr.y (mgr.34107) 905 : cluster 0 pgmap v1489: 105 pgs: 105 active+clean; 99 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 5.6 MiB/s wr, 36 op/s 2024-03-07T03:42:03.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:02 smithi189 bash[66969]: cluster 2024-03-07T03:42:01.504042+0000 mgr.y (mgr.34107) 905 : cluster 0 pgmap v1489: 105 pgs: 105 active+clean; 99 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 5.6 MiB/s wr, 36 op/s 2024-03-07T03:42:04.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:03 smithi145 bash[79878]: audit 2024-03-07T03:42:02.671486+0000 mon.a (mon.0) 1721 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:04.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:03 smithi145 bash[82329]: audit 2024-03-07T03:42:02.671486+0000 mon.a (mon.0) 1721 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:04.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:03 smithi189 bash[66969]: audit 2024-03-07T03:42:02.671486+0000 mon.a (mon.0) 1721 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:05.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:04 smithi145 bash[79878]: cluster 2024-03-07T03:42:03.504695+0000 mgr.y (mgr.34107) 906 : cluster 0 pgmap v1490: 105 pgs: 105 active+clean; 108 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 4.9 MiB/s wr, 45 op/s 2024-03-07T03:42:05.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:04 smithi145 bash[82329]: cluster 2024-03-07T03:42:03.504695+0000 mgr.y (mgr.34107) 906 : cluster 0 pgmap v1490: 105 pgs: 105 active+clean; 108 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 4.9 MiB/s wr, 45 op/s 2024-03-07T03:42:05.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:04 smithi189 bash[66969]: cluster 2024-03-07T03:42:03.504695+0000 mgr.y (mgr.34107) 906 : cluster 0 pgmap v1490: 105 pgs: 105 active+clean; 108 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 4.9 MiB/s wr, 45 op/s 2024-03-07T03:42:07.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:06 smithi145 bash[79878]: cluster 2024-03-07T03:42:05.505413+0000 mgr.y (mgr.34107) 907 : cluster 0 pgmap v1491: 105 pgs: 105 active+clean; 124 MiB data, 927 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 4.9 MiB/s wr, 37 op/s 2024-03-07T03:42:07.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:06 smithi145 bash[82329]: cluster 2024-03-07T03:42:05.505413+0000 mgr.y (mgr.34107) 907 : cluster 0 pgmap v1491: 105 pgs: 105 active+clean; 124 MiB data, 927 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 4.9 MiB/s wr, 37 op/s 2024-03-07T03:42:07.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:06 smithi189 bash[66969]: cluster 2024-03-07T03:42:05.505413+0000 mgr.y (mgr.34107) 907 : cluster 0 pgmap v1491: 105 pgs: 105 active+clean; 124 MiB data, 927 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 4.9 MiB/s wr, 37 op/s 2024-03-07T03:42:08.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:07 smithi145 bash[79878]: audit 2024-03-07T03:42:07.465060+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.0", "id": [0, 4]}]: dispatch 2024-03-07T03:42:08.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:07 smithi145 bash[79878]: audit 2024-03-07T03:42:07.465275+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.10", "id": [0, 4]}]: dispatch 2024-03-07T03:42:08.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:07 smithi145 bash[82329]: audit 2024-03-07T03:42:07.465060+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.0", "id": [0, 4]}]: dispatch 2024-03-07T03:42:08.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:07 smithi145 bash[82329]: audit 2024-03-07T03:42:07.465275+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.10", "id": [0, 4]}]: dispatch 2024-03-07T03:42:08.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:07 smithi189 bash[66969]: audit 2024-03-07T03:42:07.465060+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.0", "id": [0, 4]}]: dispatch 2024-03-07T03:42:08.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:07 smithi189 bash[66969]: audit 2024-03-07T03:42:07.465275+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.10", "id": [0, 4]}]: dispatch 2024-03-07T03:42:09.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:08 smithi145 bash[79878]: cluster 2024-03-07T03:42:07.506300+0000 mgr.y (mgr.34107) 908 : cluster 0 pgmap v1492: 105 pgs: 105 active+clean; 124 MiB data, 956 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:42:09.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:08 smithi145 bash[79878]: audit 2024-03-07T03:42:07.623032+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.0", "id": [0, 4]}]': finished 2024-03-07T03:42:09.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:08 smithi145 bash[79878]: audit 2024-03-07T03:42:07.623277+0000 mon.a (mon.0) 1725 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.10", "id": [0, 4]}]': finished 2024-03-07T03:42:09.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:08 smithi145 bash[79878]: cluster 2024-03-07T03:42:07.633577+0000 mon.a (mon.0) 1726 : cluster 0 osdmap e960: 8 total, 8 up, 8 in 2024-03-07T03:42:09.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:08 smithi145 bash[82329]: cluster 2024-03-07T03:42:07.506300+0000 mgr.y (mgr.34107) 908 : cluster 0 pgmap v1492: 105 pgs: 105 active+clean; 124 MiB data, 956 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:42:09.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:08 smithi145 bash[82329]: audit 2024-03-07T03:42:07.623032+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.0", "id": [0, 4]}]': finished 2024-03-07T03:42:09.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:08 smithi145 bash[82329]: audit 2024-03-07T03:42:07.623277+0000 mon.a (mon.0) 1725 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.10", "id": [0, 4]}]': finished 2024-03-07T03:42:09.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:08 smithi145 bash[82329]: cluster 2024-03-07T03:42:07.633577+0000 mon.a (mon.0) 1726 : cluster 0 osdmap e960: 8 total, 8 up, 8 in 2024-03-07T03:42:09.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:08 smithi189 bash[66969]: cluster 2024-03-07T03:42:07.506300+0000 mgr.y (mgr.34107) 908 : cluster 0 pgmap v1492: 105 pgs: 105 active+clean; 124 MiB data, 956 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:42:09.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:08 smithi189 bash[66969]: audit 2024-03-07T03:42:07.623032+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.0", "id": [0, 4]}]': finished 2024-03-07T03:42:09.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:08 smithi189 bash[66969]: audit 2024-03-07T03:42:07.623277+0000 mon.a (mon.0) 1725 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.10", "id": [0, 4]}]': finished 2024-03-07T03:42:09.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:08 smithi189 bash[66969]: cluster 2024-03-07T03:42:07.633577+0000 mon.a (mon.0) 1726 : cluster 0 osdmap e960: 8 total, 8 up, 8 in 2024-03-07T03:42:10.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:09 smithi145 bash[79878]: cluster 2024-03-07T03:42:08.638974+0000 mon.a (mon.0) 1727 : cluster 0 osdmap e961: 8 total, 8 up, 8 in 2024-03-07T03:42:10.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:09 smithi145 bash[82329]: cluster 2024-03-07T03:42:08.638974+0000 mon.a (mon.0) 1727 : cluster 0 osdmap e961: 8 total, 8 up, 8 in 2024-03-07T03:42:10.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:09 smithi189 bash[66969]: cluster 2024-03-07T03:42:08.638974+0000 mon.a (mon.0) 1727 : cluster 0 osdmap e961: 8 total, 8 up, 8 in 2024-03-07T03:42:11.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:10 smithi145 bash[79878]: cluster 2024-03-07T03:42:09.506955+0000 mgr.y (mgr.34107) 909 : cluster 0 pgmap v1495: 105 pgs: 105 active+clean; 132 MiB data, 980 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:42:11.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:10 smithi145 bash[82329]: cluster 2024-03-07T03:42:09.506955+0000 mgr.y (mgr.34107) 909 : cluster 0 pgmap v1495: 105 pgs: 105 active+clean; 132 MiB data, 980 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:42:11.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:10 smithi189 bash[66969]: cluster 2024-03-07T03:42:09.506955+0000 mgr.y (mgr.34107) 909 : cluster 0 pgmap v1495: 105 pgs: 105 active+clean; 132 MiB data, 980 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:42:12.582 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:42:12 smithi189 bash[59814]: ts=2024-03-07T03:42:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:42:13.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:12 smithi145 bash[79878]: cluster 2024-03-07T03:42:11.507600+0000 mgr.y (mgr.34107) 910 : cluster 0 pgmap v1496: 105 pgs: 105 active+clean; 140 MiB data, 1000 MiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 4.0 MiB/s wr, 12 op/s; 512 KiB/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:42:13.062 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:42:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:42:12] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.43.0" 2024-03-07T03:42:13.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:12 smithi145 bash[82329]: cluster 2024-03-07T03:42:11.507600+0000 mgr.y (mgr.34107) 910 : cluster 0 pgmap v1496: 105 pgs: 105 active+clean; 140 MiB data, 1000 MiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 4.0 MiB/s wr, 12 op/s; 512 KiB/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:42:13.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:12 smithi189 bash[66969]: cluster 2024-03-07T03:42:11.507600+0000 mgr.y (mgr.34107) 910 : cluster 0 pgmap v1496: 105 pgs: 105 active+clean; 140 MiB data, 1000 MiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 4.0 MiB/s wr, 12 op/s; 512 KiB/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:42:14.505 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/6.Grow (21804 ms) 2024-03-07T03:42:14.505 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/6.Shrink 2024-03-07T03:42:14.506 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:42:15.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:14 smithi145 bash[79878]: cluster 2024-03-07T03:42:13.508192+0000 mgr.y (mgr.34107) 911 : cluster 0 pgmap v1497: 105 pgs: 105 active+clean; 140 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 74 KiB/s rd, 2.0 MiB/s wr, 4 op/s; 512 KiB/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:42:15.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:14 smithi145 bash[82329]: cluster 2024-03-07T03:42:13.508192+0000 mgr.y (mgr.34107) 911 : cluster 0 pgmap v1497: 105 pgs: 105 active+clean; 140 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 74 KiB/s rd, 2.0 MiB/s wr, 4 op/s; 512 KiB/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:42:15.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:14 smithi189 bash[66969]: cluster 2024-03-07T03:42:13.508192+0000 mgr.y (mgr.34107) 911 : cluster 0 pgmap v1497: 105 pgs: 105 active+clean; 140 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 74 KiB/s rd, 2.0 MiB/s wr, 4 op/s; 512 KiB/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:42:17.061 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:16 smithi145 bash[79878]: cluster 2024-03-07T03:42:15.508929+0000 mgr.y (mgr.34107) 912 : cluster 0 pgmap v1498: 105 pgs: 105 active+clean; 153 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 4.6 MiB/s wr, 20 op/s; 512 KiB/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:42:17.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:16 smithi145 bash[82329]: cluster 2024-03-07T03:42:15.508929+0000 mgr.y (mgr.34107) 912 : cluster 0 pgmap v1498: 105 pgs: 105 active+clean; 153 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 4.6 MiB/s wr, 20 op/s; 512 KiB/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:42:17.081 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:16 smithi189 bash[66969]: cluster 2024-03-07T03:42:15.508929+0000 mgr.y (mgr.34107) 912 : cluster 0 pgmap v1498: 105 pgs: 105 active+clean; 153 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 4.6 MiB/s wr, 20 op/s; 512 KiB/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:42:19.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:18 smithi145 bash[79878]: cluster 2024-03-07T03:42:17.509624+0000 mgr.y (mgr.34107) 913 : cluster 0 pgmap v1499: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 24 op/s; 414 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:42:19.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:18 smithi145 bash[79878]: audit 2024-03-07T03:42:17.672546+0000 mon.a (mon.0) 1728 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:18 smithi145 bash[82329]: cluster 2024-03-07T03:42:17.509624+0000 mgr.y (mgr.34107) 913 : cluster 0 pgmap v1499: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 24 op/s; 414 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:42:19.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:18 smithi145 bash[82329]: audit 2024-03-07T03:42:17.672546+0000 mon.a (mon.0) 1728 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:19.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:18 smithi189 bash[66969]: cluster 2024-03-07T03:42:17.509624+0000 mgr.y (mgr.34107) 913 : cluster 0 pgmap v1499: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 24 op/s; 414 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:42:19.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:18 smithi189 bash[66969]: audit 2024-03-07T03:42:17.672546+0000 mon.a (mon.0) 1728 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:21.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:20 smithi145 bash[79878]: cluster 2024-03-07T03:42:19.510276+0000 mgr.y (mgr.34107) 914 : cluster 0 pgmap v1500: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.2 MiB/s wr, 23 op/s; 377 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:42:21.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:20 smithi145 bash[82329]: cluster 2024-03-07T03:42:19.510276+0000 mgr.y (mgr.34107) 914 : cluster 0 pgmap v1500: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.2 MiB/s wr, 23 op/s; 377 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:42:21.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:20 smithi189 bash[66969]: cluster 2024-03-07T03:42:19.510276+0000 mgr.y (mgr.34107) 914 : cluster 0 pgmap v1500: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.2 MiB/s wr, 23 op/s; 377 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:42:22.582 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:42:22 smithi189 bash[59814]: ts=2024-03-07T03:42:22.317Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:42:23.062 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:42:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:42:22] "GET /metrics HTTP/1.1" 200 36600 "" "Prometheus/2.43.0" 2024-03-07T03:42:23.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:22 smithi145 bash[79878]: cluster 2024-03-07T03:42:21.511263+0000 mgr.y (mgr.34107) 915 : cluster 0 pgmap v1501: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.1 MiB/s wr, 15 op/s; 341 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:42:23.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:22 smithi145 bash[79878]: cluster 2024-03-07T03:42:21.677775+0000 mon.a (mon.0) 1729 : cluster 0 osdmap e962: 8 total, 8 up, 8 in 2024-03-07T03:42:23.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:22 smithi145 bash[82329]: cluster 2024-03-07T03:42:21.511263+0000 mgr.y (mgr.34107) 915 : cluster 0 pgmap v1501: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.1 MiB/s wr, 15 op/s; 341 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:42:23.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:22 smithi145 bash[82329]: cluster 2024-03-07T03:42:21.677775+0000 mon.a (mon.0) 1729 : cluster 0 osdmap e962: 8 total, 8 up, 8 in 2024-03-07T03:42:23.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:22 smithi189 bash[66969]: cluster 2024-03-07T03:42:21.511263+0000 mgr.y (mgr.34107) 915 : cluster 0 pgmap v1501: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.1 MiB/s wr, 15 op/s; 341 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:42:23.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:22 smithi189 bash[66969]: cluster 2024-03-07T03:42:21.677775+0000 mon.a (mon.0) 1729 : cluster 0 osdmap e962: 8 total, 8 up, 8 in 2024-03-07T03:42:25.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:24 smithi145 bash[79878]: cluster 2024-03-07T03:42:23.511823+0000 mgr.y (mgr.34107) 916 : cluster 0 pgmap v1503: 105 pgs: 105 active+clean; 181 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 36 op/s 2024-03-07T03:42:25.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:24 smithi145 bash[82329]: cluster 2024-03-07T03:42:23.511823+0000 mgr.y (mgr.34107) 916 : cluster 0 pgmap v1503: 105 pgs: 105 active+clean; 181 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 36 op/s 2024-03-07T03:42:25.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:24 smithi189 bash[66969]: cluster 2024-03-07T03:42:23.511823+0000 mgr.y (mgr.34107) 916 : cluster 0 pgmap v1503: 105 pgs: 105 active+clean; 181 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 36 op/s 2024-03-07T03:42:27.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:26 smithi145 bash[79878]: cluster 2024-03-07T03:42:25.512862+0000 mgr.y (mgr.34107) 917 : cluster 0 pgmap v1504: 105 pgs: 105 active+clean; 193 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.9 MiB/s rd, 4.8 MiB/s wr, 21 op/s 2024-03-07T03:42:27.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:26 smithi145 bash[82329]: cluster 2024-03-07T03:42:25.512862+0000 mgr.y (mgr.34107) 917 : cluster 0 pgmap v1504: 105 pgs: 105 active+clean; 193 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.9 MiB/s rd, 4.8 MiB/s wr, 21 op/s 2024-03-07T03:42:27.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:26 smithi189 bash[66969]: cluster 2024-03-07T03:42:25.512862+0000 mgr.y (mgr.34107) 917 : cluster 0 pgmap v1504: 105 pgs: 105 active+clean; 193 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.9 MiB/s rd, 4.8 MiB/s wr, 21 op/s 2024-03-07T03:42:29.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:28 smithi145 bash[82329]: cluster 2024-03-07T03:42:27.513595+0000 mgr.y (mgr.34107) 918 : cluster 0 pgmap v1505: 105 pgs: 105 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:42:29.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:28 smithi145 bash[79878]: cluster 2024-03-07T03:42:27.513595+0000 mgr.y (mgr.34107) 918 : cluster 0 pgmap v1505: 105 pgs: 105 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:42:29.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:28 smithi189 bash[66969]: cluster 2024-03-07T03:42:27.513595+0000 mgr.y (mgr.34107) 918 : cluster 0 pgmap v1505: 105 pgs: 105 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:42:31.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:30 smithi145 bash[79878]: cluster 2024-03-07T03:42:29.514240+0000 mgr.y (mgr.34107) 919 : cluster 0 pgmap v1506: 105 pgs: 105 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:42:31.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:30 smithi145 bash[82329]: cluster 2024-03-07T03:42:29.514240+0000 mgr.y (mgr.34107) 919 : cluster 0 pgmap v1506: 105 pgs: 105 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:42:31.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:30 smithi189 bash[66969]: cluster 2024-03-07T03:42:29.514240+0000 mgr.y (mgr.34107) 919 : cluster 0 pgmap v1506: 105 pgs: 105 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:42:32.582 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:42:32 smithi189 bash[59814]: ts=2024-03-07T03:42:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:42:33.062 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:42:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:42:32] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.43.0" 2024-03-07T03:42:33.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:32 smithi145 bash[79878]: cluster 2024-03-07T03:42:31.514892+0000 mgr.y (mgr.34107) 920 : cluster 0 pgmap v1507: 105 pgs: 105 active+clean; 210 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-03-07T03:42:33.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:32 smithi145 bash[79878]: audit 2024-03-07T03:42:32.672623+0000 mon.a (mon.0) 1730 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:33.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:32 smithi145 bash[82329]: cluster 2024-03-07T03:42:31.514892+0000 mgr.y (mgr.34107) 920 : cluster 0 pgmap v1507: 105 pgs: 105 active+clean; 210 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-03-07T03:42:33.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:32 smithi145 bash[82329]: audit 2024-03-07T03:42:32.672623+0000 mon.a (mon.0) 1730 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:33.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:32 smithi189 bash[66969]: cluster 2024-03-07T03:42:31.514892+0000 mgr.y (mgr.34107) 920 : cluster 0 pgmap v1507: 105 pgs: 105 active+clean; 210 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-03-07T03:42:33.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:32 smithi189 bash[66969]: audit 2024-03-07T03:42:32.672623+0000 mon.a (mon.0) 1730 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:35.012 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/6.Shrink (20506 ms) 2024-03-07T03:42:35.012 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/6.ShrinkToOne 2024-03-07T03:42:35.012 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:42:35.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:34 smithi145 bash[79878]: cluster 2024-03-07T03:42:33.515542+0000 mgr.y (mgr.34107) 921 : cluster 0 pgmap v1508: 105 pgs: 105 active+clean; 218 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 3.8 MiB/s wr, 37 op/s 2024-03-07T03:42:35.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:34 smithi145 bash[82329]: cluster 2024-03-07T03:42:33.515542+0000 mgr.y (mgr.34107) 921 : cluster 0 pgmap v1508: 105 pgs: 105 active+clean; 218 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 3.8 MiB/s wr, 37 op/s 2024-03-07T03:42:35.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:34 smithi189 bash[66969]: cluster 2024-03-07T03:42:33.515542+0000 mgr.y (mgr.34107) 921 : cluster 0 pgmap v1508: 105 pgs: 105 active+clean; 218 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 3.8 MiB/s wr, 37 op/s 2024-03-07T03:42:37.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:36 smithi145 bash[79878]: cluster 2024-03-07T03:42:35.516580+0000 mgr.y (mgr.34107) 922 : cluster 0 pgmap v1509: 105 pgs: 105 active+clean; 235 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 4.8 MiB/s wr, 32 op/s 2024-03-07T03:42:37.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:36 smithi145 bash[82329]: cluster 2024-03-07T03:42:35.516580+0000 mgr.y (mgr.34107) 922 : cluster 0 pgmap v1509: 105 pgs: 105 active+clean; 235 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 4.8 MiB/s wr, 32 op/s 2024-03-07T03:42:37.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:36 smithi189 bash[66969]: cluster 2024-03-07T03:42:35.516580+0000 mgr.y (mgr.34107) 922 : cluster 0 pgmap v1509: 105 pgs: 105 active+clean; 235 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 4.8 MiB/s wr, 32 op/s 2024-03-07T03:42:39.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:38 smithi145 bash[82329]: cluster 2024-03-07T03:42:37.517329+0000 mgr.y (mgr.34107) 923 : cluster 0 pgmap v1510: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 5.8 MiB/s wr, 46 op/s 2024-03-07T03:42:39.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:38 smithi145 bash[79878]: cluster 2024-03-07T03:42:37.517329+0000 mgr.y (mgr.34107) 923 : cluster 0 pgmap v1510: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 5.8 MiB/s wr, 46 op/s 2024-03-07T03:42:39.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:38 smithi189 bash[66969]: cluster 2024-03-07T03:42:37.517329+0000 mgr.y (mgr.34107) 923 : cluster 0 pgmap v1510: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 5.8 MiB/s wr, 46 op/s 2024-03-07T03:42:41.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:40 smithi145 bash[79878]: cluster 2024-03-07T03:42:39.517966+0000 mgr.y (mgr.34107) 924 : cluster 0 pgmap v1511: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-03-07T03:42:41.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:40 smithi145 bash[82329]: cluster 2024-03-07T03:42:39.517966+0000 mgr.y (mgr.34107) 924 : cluster 0 pgmap v1511: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-03-07T03:42:41.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:40 smithi189 bash[66969]: cluster 2024-03-07T03:42:39.517966+0000 mgr.y (mgr.34107) 924 : cluster 0 pgmap v1511: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-03-07T03:42:42.582 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:42:42 smithi189 bash[59814]: ts=2024-03-07T03:42:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:42:43.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:42 smithi145 bash[79878]: cluster 2024-03-07T03:42:41.518578+0000 mgr.y (mgr.34107) 925 : cluster 0 pgmap v1512: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-03-07T03:42:43.062 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:42:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:42:42] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.43.0" 2024-03-07T03:42:43.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:42 smithi145 bash[82329]: cluster 2024-03-07T03:42:41.518578+0000 mgr.y (mgr.34107) 925 : cluster 0 pgmap v1512: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-03-07T03:42:43.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:42 smithi189 bash[66969]: cluster 2024-03-07T03:42:41.518578+0000 mgr.y (mgr.34107) 925 : cluster 0 pgmap v1512: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-03-07T03:42:44.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:43 smithi145 bash[79878]: cluster 2024-03-07T03:42:42.724458+0000 mon.a (mon.0) 1731 : cluster 0 osdmap e963: 8 total, 8 up, 8 in 2024-03-07T03:42:44.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:43 smithi145 bash[82329]: cluster 2024-03-07T03:42:42.724458+0000 mon.a (mon.0) 1731 : cluster 0 osdmap e963: 8 total, 8 up, 8 in 2024-03-07T03:42:44.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:43 smithi189 bash[66969]: cluster 2024-03-07T03:42:42.724458+0000 mon.a (mon.0) 1731 : cluster 0 osdmap e963: 8 total, 8 up, 8 in 2024-03-07T03:42:45.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:44 smithi145 bash[82329]: cluster 2024-03-07T03:42:43.519175+0000 mgr.y (mgr.34107) 926 : cluster 0 pgmap v1514: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.5 MiB/s wr, 26 op/s 2024-03-07T03:42:45.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:44 smithi145 bash[79878]: cluster 2024-03-07T03:42:43.519175+0000 mgr.y (mgr.34107) 926 : cluster 0 pgmap v1514: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.5 MiB/s wr, 26 op/s 2024-03-07T03:42:45.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:44 smithi189 bash[66969]: cluster 2024-03-07T03:42:43.519175+0000 mgr.y (mgr.34107) 926 : cluster 0 pgmap v1514: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.5 MiB/s wr, 26 op/s 2024-03-07T03:42:47.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:46 smithi145 bash[79878]: cluster 2024-03-07T03:42:45.519952+0000 mgr.y (mgr.34107) 927 : cluster 0 pgmap v1515: 105 pgs: 105 active+clean; 260 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.7 MiB/s wr, 46 op/s 2024-03-07T03:42:47.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:46 smithi145 bash[82329]: cluster 2024-03-07T03:42:45.519952+0000 mgr.y (mgr.34107) 927 : cluster 0 pgmap v1515: 105 pgs: 105 active+clean; 260 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.7 MiB/s wr, 46 op/s 2024-03-07T03:42:47.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:46 smithi189 bash[66969]: cluster 2024-03-07T03:42:45.519952+0000 mgr.y (mgr.34107) 927 : cluster 0 pgmap v1515: 105 pgs: 105 active+clean; 260 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.7 MiB/s wr, 46 op/s 2024-03-07T03:42:48.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:47 smithi145 bash[79878]: audit 2024-03-07T03:42:47.672447+0000 mon.a (mon.0) 1732 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:48.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:47 smithi145 bash[82329]: audit 2024-03-07T03:42:47.672447+0000 mon.a (mon.0) 1732 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:48.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:47 smithi189 bash[66969]: audit 2024-03-07T03:42:47.672447+0000 mon.a (mon.0) 1732 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:49.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:48 smithi145 bash[79878]: cluster 2024-03-07T03:42:47.520659+0000 mgr.y (mgr.34107) 928 : cluster 0 pgmap v1516: 105 pgs: 105 active+clean; 288 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:42:49.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:48 smithi145 bash[79878]: audit 2024-03-07T03:42:48.007088+0000 mon.a (mon.0) 1733 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:42:49.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:48 smithi145 bash[82329]: cluster 2024-03-07T03:42:47.520659+0000 mgr.y (mgr.34107) 928 : cluster 0 pgmap v1516: 105 pgs: 105 active+clean; 288 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:42:49.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:48 smithi145 bash[82329]: audit 2024-03-07T03:42:48.007088+0000 mon.a (mon.0) 1733 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:42:49.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:48 smithi189 bash[66969]: cluster 2024-03-07T03:42:47.520659+0000 mgr.y (mgr.34107) 928 : cluster 0 pgmap v1516: 105 pgs: 105 active+clean; 288 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:42:49.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:48 smithi189 bash[66969]: audit 2024-03-07T03:42:48.007088+0000 mon.a (mon.0) 1733 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:42:51.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:50 smithi145 bash[79878]: cluster 2024-03-07T03:42:49.521326+0000 mgr.y (mgr.34107) 929 : cluster 0 pgmap v1517: 105 pgs: 105 active+clean; 288 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:42:51.062 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:50 smithi145 bash[82329]: cluster 2024-03-07T03:42:49.521326+0000 mgr.y (mgr.34107) 929 : cluster 0 pgmap v1517: 105 pgs: 105 active+clean; 288 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:42:51.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:50 smithi189 bash[66969]: cluster 2024-03-07T03:42:49.521326+0000 mgr.y (mgr.34107) 929 : cluster 0 pgmap v1517: 105 pgs: 105 active+clean; 288 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:42:52.582 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:42:52 smithi189 bash[59814]: ts=2024-03-07T03:42:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:42:53.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:52 smithi145 bash[79878]: cluster 2024-03-07T03:42:51.521991+0000 mgr.y (mgr.34107) 930 : cluster 0 pgmap v1518: 105 pgs: 105 active+clean; 292 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.5 MiB/s wr, 39 op/s 2024-03-07T03:42:53.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:52 smithi145 bash[82329]: cluster 2024-03-07T03:42:51.521991+0000 mgr.y (mgr.34107) 930 : cluster 0 pgmap v1518: 105 pgs: 105 active+clean; 292 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.5 MiB/s wr, 39 op/s 2024-03-07T03:42:53.063 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:42:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:42:52] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.43.0" 2024-03-07T03:42:53.082 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:52 smithi189 bash[66969]: cluster 2024-03-07T03:42:51.521991+0000 mgr.y (mgr.34107) 930 : cluster 0 pgmap v1518: 105 pgs: 105 active+clean; 292 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.5 MiB/s wr, 39 op/s 2024-03-07T03:42:55.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[79878]: cluster 2024-03-07T03:42:53.522643+0000 mgr.y (mgr.34107) 931 : cluster 0 pgmap v1519: 105 pgs: 105 active+clean; 296 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:42:55.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[79878]: audit 2024-03-07T03:42:53.942243+0000 mon.a (mon.0) 1734 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[79878]: audit 2024-03-07T03:42:53.949570+0000 mon.a (mon.0) 1735 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[79878]: audit 2024-03-07T03:42:53.997594+0000 mon.a (mon.0) 1736 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[79878]: audit 2024-03-07T03:42:54.011216+0000 mon.a (mon.0) 1737 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[79878]: audit 2024-03-07T03:42:54.634396+0000 mon.a (mon.0) 1738 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:42:55.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[79878]: audit 2024-03-07T03:42:54.635382+0000 mon.a (mon.0) 1739 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:42:55.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[79878]: audit 2024-03-07T03:42:54.640792+0000 mon.a (mon.0) 1740 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[82329]: cluster 2024-03-07T03:42:53.522643+0000 mgr.y (mgr.34107) 931 : cluster 0 pgmap v1519: 105 pgs: 105 active+clean; 296 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:42:55.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[82329]: audit 2024-03-07T03:42:53.942243+0000 mon.a (mon.0) 1734 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[82329]: audit 2024-03-07T03:42:53.949570+0000 mon.a (mon.0) 1735 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[82329]: audit 2024-03-07T03:42:53.997594+0000 mon.a (mon.0) 1736 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[82329]: audit 2024-03-07T03:42:54.011216+0000 mon.a (mon.0) 1737 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.314 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[82329]: audit 2024-03-07T03:42:54.634396+0000 mon.a (mon.0) 1738 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:42:55.314 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[82329]: audit 2024-03-07T03:42:54.635382+0000 mon.a (mon.0) 1739 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:42:55.314 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:54 smithi145 bash[82329]: audit 2024-03-07T03:42:54.640792+0000 mon.a (mon.0) 1740 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:54 smithi189 bash[66969]: cluster 2024-03-07T03:42:53.522643+0000 mgr.y (mgr.34107) 931 : cluster 0 pgmap v1519: 105 pgs: 105 active+clean; 296 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:42:55.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:54 smithi189 bash[66969]: audit 2024-03-07T03:42:53.942243+0000 mon.a (mon.0) 1734 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:54 smithi189 bash[66969]: audit 2024-03-07T03:42:53.949570+0000 mon.a (mon.0) 1735 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:54 smithi189 bash[66969]: audit 2024-03-07T03:42:53.997594+0000 mon.a (mon.0) 1736 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:54 smithi189 bash[66969]: audit 2024-03-07T03:42:54.011216+0000 mon.a (mon.0) 1737 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:55.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:54 smithi189 bash[66969]: audit 2024-03-07T03:42:54.634396+0000 mon.a (mon.0) 1738 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:42:55.334 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:54 smithi189 bash[66969]: audit 2024-03-07T03:42:54.635382+0000 mon.a (mon.0) 1739 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:42:55.334 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:54 smithi189 bash[66969]: audit 2024-03-07T03:42:54.640792+0000 mon.a (mon.0) 1740 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:42:56.276 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/6.ShrinkToOne (21264 ms) 2024-03-07T03:42:56.277 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/6.ShrinkToOneAfterSnapshot 2024-03-07T03:42:56.277 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:42:57.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:56 smithi145 bash[79878]: cluster 2024-03-07T03:42:55.523325+0000 mgr.y (mgr.34107) 932 : cluster 0 pgmap v1520: 105 pgs: 105 active+clean; 304 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 4.8 MiB/s wr, 44 op/s 2024-03-07T03:42:57.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:56 smithi145 bash[82329]: cluster 2024-03-07T03:42:55.523325+0000 mgr.y (mgr.34107) 932 : cluster 0 pgmap v1520: 105 pgs: 105 active+clean; 304 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 4.8 MiB/s wr, 44 op/s 2024-03-07T03:42:57.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:56 smithi189 bash[66969]: cluster 2024-03-07T03:42:55.523325+0000 mgr.y (mgr.34107) 932 : cluster 0 pgmap v1520: 105 pgs: 105 active+clean; 304 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 4.8 MiB/s wr, 44 op/s 2024-03-07T03:42:59.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:42:58 smithi145 bash[79878]: cluster 2024-03-07T03:42:57.523994+0000 mgr.y (mgr.34107) 933 : cluster 0 pgmap v1521: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 MiB/s wr, 28 op/s 2024-03-07T03:42:59.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:42:58 smithi145 bash[82329]: cluster 2024-03-07T03:42:57.523994+0000 mgr.y (mgr.34107) 933 : cluster 0 pgmap v1521: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 MiB/s wr, 28 op/s 2024-03-07T03:42:59.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:42:58 smithi189 bash[66969]: cluster 2024-03-07T03:42:57.523994+0000 mgr.y (mgr.34107) 933 : cluster 0 pgmap v1521: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.0 MiB/s wr, 28 op/s 2024-03-07T03:43:01.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:00 smithi145 bash[79878]: cluster 2024-03-07T03:42:59.524662+0000 mgr.y (mgr.34107) 934 : cluster 0 pgmap v1522: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 21 op/s 2024-03-07T03:43:01.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:00 smithi145 bash[82329]: cluster 2024-03-07T03:42:59.524662+0000 mgr.y (mgr.34107) 934 : cluster 0 pgmap v1522: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 21 op/s 2024-03-07T03:43:01.332 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:00 smithi189 bash[66969]: cluster 2024-03-07T03:42:59.524662+0000 mgr.y (mgr.34107) 934 : cluster 0 pgmap v1522: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 21 op/s 2024-03-07T03:43:02.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:43:02 smithi189 bash[59814]: ts=2024-03-07T03:43:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:43:03.062 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:02 smithi145 bash[79878]: cluster 2024-03-07T03:43:01.525340+0000 mgr.y (mgr.34107) 935 : cluster 0 pgmap v1523: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-03-07T03:43:03.063 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:02 smithi145 bash[79878]: audit 2024-03-07T03:43:02.672765+0000 mon.a (mon.0) 1741 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:03.063 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:43:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:43:02] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.43.0" 2024-03-07T03:43:03.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:02 smithi145 bash[82329]: cluster 2024-03-07T03:43:01.525340+0000 mgr.y (mgr.34107) 935 : cluster 0 pgmap v1523: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-03-07T03:43:03.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:02 smithi145 bash[82329]: audit 2024-03-07T03:43:02.672765+0000 mon.a (mon.0) 1741 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:03.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:02 smithi189 bash[66969]: cluster 2024-03-07T03:43:01.525340+0000 mgr.y (mgr.34107) 935 : cluster 0 pgmap v1523: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-03-07T03:43:03.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:02 smithi189 bash[66969]: audit 2024-03-07T03:43:02.672765+0000 mon.a (mon.0) 1741 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:05.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:04 smithi145 bash[79878]: cluster 2024-03-07T03:43:03.525977+0000 mgr.y (mgr.34107) 936 : cluster 0 pgmap v1524: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-03-07T03:43:05.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:04 smithi145 bash[82329]: cluster 2024-03-07T03:43:03.525977+0000 mgr.y (mgr.34107) 936 : cluster 0 pgmap v1524: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-03-07T03:43:05.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:04 smithi189 bash[66969]: cluster 2024-03-07T03:43:03.525977+0000 mgr.y (mgr.34107) 936 : cluster 0 pgmap v1524: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-03-07T03:43:06.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:05 smithi145 bash[79878]: cluster 2024-03-07T03:43:04.979067+0000 mon.a (mon.0) 1742 : cluster 0 osdmap e964: 8 total, 8 up, 8 in 2024-03-07T03:43:06.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:05 smithi145 bash[82329]: cluster 2024-03-07T03:43:04.979067+0000 mon.a (mon.0) 1742 : cluster 0 osdmap e964: 8 total, 8 up, 8 in 2024-03-07T03:43:06.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:05 smithi189 bash[66969]: cluster 2024-03-07T03:43:04.979067+0000 mon.a (mon.0) 1742 : cluster 0 osdmap e964: 8 total, 8 up, 8 in 2024-03-07T03:43:07.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:06 smithi145 bash[79878]: cluster 2024-03-07T03:43:05.526543+0000 mgr.y (mgr.34107) 937 : cluster 0 pgmap v1526: 105 pgs: 105 active+clean; 312 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 9 op/s 2024-03-07T03:43:07.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:06 smithi145 bash[82329]: cluster 2024-03-07T03:43:05.526543+0000 mgr.y (mgr.34107) 937 : cluster 0 pgmap v1526: 105 pgs: 105 active+clean; 312 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 9 op/s 2024-03-07T03:43:07.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:06 smithi189 bash[66969]: cluster 2024-03-07T03:43:05.526543+0000 mgr.y (mgr.34107) 937 : cluster 0 pgmap v1526: 105 pgs: 105 active+clean; 312 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 9 op/s 2024-03-07T03:43:09.312 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:08 smithi145 bash[79878]: cluster 2024-03-07T03:43:07.527341+0000 mgr.y (mgr.34107) 938 : cluster 0 pgmap v1527: 105 pgs: 105 active+clean; 341 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 858 KiB/s rd, 4.1 MiB/s wr, 18 op/s 2024-03-07T03:43:09.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:08 smithi145 bash[82329]: cluster 2024-03-07T03:43:07.527341+0000 mgr.y (mgr.34107) 938 : cluster 0 pgmap v1527: 105 pgs: 105 active+clean; 341 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 858 KiB/s rd, 4.1 MiB/s wr, 18 op/s 2024-03-07T03:43:09.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:08 smithi189 bash[66969]: cluster 2024-03-07T03:43:07.527341+0000 mgr.y (mgr.34107) 938 : cluster 0 pgmap v1527: 105 pgs: 105 active+clean; 341 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 858 KiB/s rd, 4.1 MiB/s wr, 18 op/s 2024-03-07T03:43:11.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:10 smithi145 bash[79878]: cluster 2024-03-07T03:43:09.528014+0000 mgr.y (mgr.34107) 939 : cluster 0 pgmap v1528: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 37 op/s 2024-03-07T03:43:11.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:10 smithi145 bash[82329]: cluster 2024-03-07T03:43:09.528014+0000 mgr.y (mgr.34107) 939 : cluster 0 pgmap v1528: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 37 op/s 2024-03-07T03:43:11.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:10 smithi189 bash[66969]: cluster 2024-03-07T03:43:09.528014+0000 mgr.y (mgr.34107) 939 : cluster 0 pgmap v1528: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 37 op/s 2024-03-07T03:43:12.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:43:12 smithi189 bash[59814]: ts=2024-03-07T03:43:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:43:13.063 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:12 smithi145 bash[79878]: cluster 2024-03-07T03:43:11.528649+0000 mgr.y (mgr.34107) 940 : cluster 0 pgmap v1529: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:43:13.063 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:43:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:43:12] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.43.0" 2024-03-07T03:43:13.063 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:12 smithi145 bash[82329]: cluster 2024-03-07T03:43:11.528649+0000 mgr.y (mgr.34107) 940 : cluster 0 pgmap v1529: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:43:13.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:12 smithi189 bash[66969]: cluster 2024-03-07T03:43:11.528649+0000 mgr.y (mgr.34107) 940 : cluster 0 pgmap v1529: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:43:15.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:14 smithi145 bash[79878]: cluster 2024-03-07T03:43:13.529243+0000 mgr.y (mgr.34107) 941 : cluster 0 pgmap v1530: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:43:15.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:14 smithi145 bash[82329]: cluster 2024-03-07T03:43:13.529243+0000 mgr.y (mgr.34107) 941 : cluster 0 pgmap v1530: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:43:15.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:14 smithi189 bash[66969]: cluster 2024-03-07T03:43:13.529243+0000 mgr.y (mgr.34107) 941 : cluster 0 pgmap v1530: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:43:17.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:16 smithi145 bash[79878]: cluster 2024-03-07T03:43:15.529968+0000 mgr.y (mgr.34107) 942 : cluster 0 pgmap v1531: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.5 MiB/s wr, 33 op/s 2024-03-07T03:43:17.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:17 smithi145 bash[82329]: cluster 2024-03-07T03:43:15.529968+0000 mgr.y (mgr.34107) 942 : cluster 0 pgmap v1531: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.5 MiB/s wr, 33 op/s 2024-03-07T03:43:17.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:17 smithi189 bash[66969]: cluster 2024-03-07T03:43:15.529968+0000 mgr.y (mgr.34107) 942 : cluster 0 pgmap v1531: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.5 MiB/s wr, 33 op/s 2024-03-07T03:43:18.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:18 smithi145 bash[79878]: audit 2024-03-07T03:43:17.673093+0000 mon.a (mon.0) 1743 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:18.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:18 smithi145 bash[82329]: audit 2024-03-07T03:43:17.673093+0000 mon.a (mon.0) 1743 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:18.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:18 smithi189 bash[66969]: audit 2024-03-07T03:43:17.673093+0000 mon.a (mon.0) 1743 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:19.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:19 smithi145 bash[82329]: cluster 2024-03-07T03:43:17.530905+0000 mgr.y (mgr.34107) 943 : cluster 0 pgmap v1532: 105 pgs: 105 active+clean; 357 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.8 MiB/s wr, 33 op/s 2024-03-07T03:43:19.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:19 smithi145 bash[79878]: cluster 2024-03-07T03:43:17.530905+0000 mgr.y (mgr.34107) 943 : cluster 0 pgmap v1532: 105 pgs: 105 active+clean; 357 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.8 MiB/s wr, 33 op/s 2024-03-07T03:43:19.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:19 smithi189 bash[66969]: cluster 2024-03-07T03:43:17.530905+0000 mgr.y (mgr.34107) 943 : cluster 0 pgmap v1532: 105 pgs: 105 active+clean; 357 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.8 MiB/s wr, 33 op/s 2024-03-07T03:43:21.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:21 smithi145 bash[79878]: cluster 2024-03-07T03:43:19.531806+0000 mgr.y (mgr.34107) 944 : cluster 0 pgmap v1533: 105 pgs: 105 active+clean; 365 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-03-07T03:43:21.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:21 smithi145 bash[79878]: cluster 2024-03-07T03:43:20.022224+0000 mon.a (mon.0) 1744 : cluster 0 osdmap e965: 8 total, 8 up, 8 in 2024-03-07T03:43:21.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:21 smithi145 bash[82329]: cluster 2024-03-07T03:43:19.531806+0000 mgr.y (mgr.34107) 944 : cluster 0 pgmap v1533: 105 pgs: 105 active+clean; 365 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-03-07T03:43:21.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:21 smithi145 bash[82329]: cluster 2024-03-07T03:43:20.022224+0000 mon.a (mon.0) 1744 : cluster 0 osdmap e965: 8 total, 8 up, 8 in 2024-03-07T03:43:21.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:21 smithi189 bash[66969]: cluster 2024-03-07T03:43:19.531806+0000 mgr.y (mgr.34107) 944 : cluster 0 pgmap v1533: 105 pgs: 105 active+clean; 365 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-03-07T03:43:21.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:21 smithi189 bash[66969]: cluster 2024-03-07T03:43:20.022224+0000 mon.a (mon.0) 1744 : cluster 0 osdmap e965: 8 total, 8 up, 8 in 2024-03-07T03:43:22.452 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/6.ShrinkToOneAfterSnapshot (26175 ms) 2024-03-07T03:43:22.452 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/6.MinOverlap 2024-03-07T03:43:22.452 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:43:22.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:43:22 smithi189 bash[59814]: ts=2024-03-07T03:43:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:43:23.013 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:43:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:43:22] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.43.0" 2024-03-07T03:43:23.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:23 smithi145 bash[79878]: cluster 2024-03-07T03:43:21.532440+0000 mgr.y (mgr.34107) 945 : cluster 0 pgmap v1535: 105 pgs: 105 active+clean; 381 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-03-07T03:43:23.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:23 smithi145 bash[82329]: cluster 2024-03-07T03:43:21.532440+0000 mgr.y (mgr.34107) 945 : cluster 0 pgmap v1535: 105 pgs: 105 active+clean; 381 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-03-07T03:43:23.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:23 smithi189 bash[66969]: cluster 2024-03-07T03:43:21.532440+0000 mgr.y (mgr.34107) 945 : cluster 0 pgmap v1535: 105 pgs: 105 active+clean; 381 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-03-07T03:43:25.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:25 smithi145 bash[79878]: cluster 2024-03-07T03:43:23.533146+0000 mgr.y (mgr.34107) 946 : cluster 0 pgmap v1536: 105 pgs: 105 active+clean; 402 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 5.3 MiB/s wr, 29 op/s 2024-03-07T03:43:25.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:25 smithi145 bash[82329]: cluster 2024-03-07T03:43:23.533146+0000 mgr.y (mgr.34107) 946 : cluster 0 pgmap v1536: 105 pgs: 105 active+clean; 402 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 5.3 MiB/s wr, 29 op/s 2024-03-07T03:43:25.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:25 smithi189 bash[66969]: cluster 2024-03-07T03:43:23.533146+0000 mgr.y (mgr.34107) 946 : cluster 0 pgmap v1536: 105 pgs: 105 active+clean; 402 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 5.3 MiB/s wr, 29 op/s 2024-03-07T03:43:27.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:27 smithi145 bash[79878]: cluster 2024-03-07T03:43:25.533881+0000 mgr.y (mgr.34107) 947 : cluster 0 pgmap v1537: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.5 MiB/s wr, 45 op/s 2024-03-07T03:43:27.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:27 smithi145 bash[82329]: cluster 2024-03-07T03:43:25.533881+0000 mgr.y (mgr.34107) 947 : cluster 0 pgmap v1537: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.5 MiB/s wr, 45 op/s 2024-03-07T03:43:27.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:27 smithi189 bash[66969]: cluster 2024-03-07T03:43:25.533881+0000 mgr.y (mgr.34107) 947 : cluster 0 pgmap v1537: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.5 MiB/s wr, 45 op/s 2024-03-07T03:43:29.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:29 smithi145 bash[79878]: cluster 2024-03-07T03:43:27.534585+0000 mgr.y (mgr.34107) 948 : cluster 0 pgmap v1538: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.7 MiB/s wr, 45 op/s 2024-03-07T03:43:29.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:29 smithi145 bash[82329]: cluster 2024-03-07T03:43:27.534585+0000 mgr.y (mgr.34107) 948 : cluster 0 pgmap v1538: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.7 MiB/s wr, 45 op/s 2024-03-07T03:43:29.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:29 smithi189 bash[66969]: cluster 2024-03-07T03:43:27.534585+0000 mgr.y (mgr.34107) 948 : cluster 0 pgmap v1538: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.7 MiB/s wr, 45 op/s 2024-03-07T03:43:31.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:31 smithi145 bash[79878]: cluster 2024-03-07T03:43:29.535237+0000 mgr.y (mgr.34107) 949 : cluster 0 pgmap v1539: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 6.9 MiB/s wr, 38 op/s 2024-03-07T03:43:31.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:31 smithi145 bash[82329]: cluster 2024-03-07T03:43:29.535237+0000 mgr.y (mgr.34107) 949 : cluster 0 pgmap v1539: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 6.9 MiB/s wr, 38 op/s 2024-03-07T03:43:31.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:31 smithi189 bash[66969]: cluster 2024-03-07T03:43:29.535237+0000 mgr.y (mgr.34107) 949 : cluster 0 pgmap v1539: 105 pgs: 105 active+clean; 434 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 6.9 MiB/s wr, 38 op/s 2024-03-07T03:43:32.308 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:32 smithi189 bash[66969]: cluster 2024-03-07T03:43:31.063313+0000 mon.a (mon.0) 1745 : cluster 0 osdmap e966: 8 total, 8 up, 8 in 2024-03-07T03:43:32.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:32 smithi145 bash[79878]: cluster 2024-03-07T03:43:31.063313+0000 mon.a (mon.0) 1745 : cluster 0 osdmap e966: 8 total, 8 up, 8 in 2024-03-07T03:43:32.313 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:32 smithi145 bash[82329]: cluster 2024-03-07T03:43:31.063313+0000 mon.a (mon.0) 1745 : cluster 0 osdmap e966: 8 total, 8 up, 8 in 2024-03-07T03:43:32.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:43:32 smithi189 bash[59814]: ts=2024-03-07T03:43:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:43:33.058 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:43:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:43:32] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.43.0" 2024-03-07T03:43:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:33 smithi145 bash[82329]: cluster 2024-03-07T03:43:31.535783+0000 mgr.y (mgr.34107) 950 : cluster 0 pgmap v1541: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 5.3 MiB/s wr, 34 op/s 2024-03-07T03:43:33.059 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:33 smithi145 bash[82329]: audit 2024-03-07T03:43:32.673719+0000 mon.a (mon.0) 1746 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:33.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:33 smithi145 bash[79878]: cluster 2024-03-07T03:43:31.535783+0000 mgr.y (mgr.34107) 950 : cluster 0 pgmap v1541: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 5.3 MiB/s wr, 34 op/s 2024-03-07T03:43:33.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:33 smithi145 bash[79878]: audit 2024-03-07T03:43:32.673719+0000 mon.a (mon.0) 1746 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:33.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:33 smithi189 bash[66969]: cluster 2024-03-07T03:43:31.535783+0000 mgr.y (mgr.34107) 950 : cluster 0 pgmap v1541: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 5.3 MiB/s wr, 34 op/s 2024-03-07T03:43:33.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:33 smithi189 bash[66969]: audit 2024-03-07T03:43:32.673719+0000 mon.a (mon.0) 1746 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:35.313 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:35 smithi145 bash[79878]: cluster 2024-03-07T03:43:33.536521+0000 mgr.y (mgr.34107) 951 : cluster 0 pgmap v1542: 105 pgs: 105 active+clean; 447 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 4.5 MiB/s wr, 33 op/s 2024-03-07T03:43:35.314 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:35 smithi145 bash[82329]: cluster 2024-03-07T03:43:33.536521+0000 mgr.y (mgr.34107) 951 : cluster 0 pgmap v1542: 105 pgs: 105 active+clean; 447 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 4.5 MiB/s wr, 33 op/s 2024-03-07T03:43:35.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:35 smithi189 bash[66969]: cluster 2024-03-07T03:43:33.536521+0000 mgr.y (mgr.34107) 951 : cluster 0 pgmap v1542: 105 pgs: 105 active+clean; 447 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 4.5 MiB/s wr, 33 op/s 2024-03-07T03:43:37.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:37 smithi189 bash[66969]: cluster 2024-03-07T03:43:35.537235+0000 mgr.y (mgr.34107) 952 : cluster 0 pgmap v1543: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-03-07T03:43:37.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:37 smithi145 bash[79878]: cluster 2024-03-07T03:43:35.537235+0000 mgr.y (mgr.34107) 952 : cluster 0 pgmap v1543: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-03-07T03:43:37.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:37 smithi145 bash[82329]: cluster 2024-03-07T03:43:35.537235+0000 mgr.y (mgr.34107) 952 : cluster 0 pgmap v1543: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-03-07T03:43:39.334 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:39 smithi189 bash[66969]: cluster 2024-03-07T03:43:37.537931+0000 mgr.y (mgr.34107) 953 : cluster 0 pgmap v1544: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:43:39.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:39 smithi145 bash[79878]: cluster 2024-03-07T03:43:37.537931+0000 mgr.y (mgr.34107) 953 : cluster 0 pgmap v1544: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:43:39.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:39 smithi145 bash[82329]: cluster 2024-03-07T03:43:37.537931+0000 mgr.y (mgr.34107) 953 : cluster 0 pgmap v1544: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:43:41.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:41 smithi189 bash[66969]: cluster 2024-03-07T03:43:39.538584+0000 mgr.y (mgr.34107) 954 : cluster 0 pgmap v1545: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-03-07T03:43:41.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:41 smithi145 bash[79878]: cluster 2024-03-07T03:43:39.538584+0000 mgr.y (mgr.34107) 954 : cluster 0 pgmap v1545: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-03-07T03:43:41.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:41 smithi145 bash[82329]: cluster 2024-03-07T03:43:39.538584+0000 mgr.y (mgr.34107) 954 : cluster 0 pgmap v1545: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-03-07T03:43:42.583 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:43:42 smithi189 bash[59814]: ts=2024-03-07T03:43:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:43:43.064 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:43:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:43:42] "GET /metrics HTTP/1.1" 200 36594 "" "Prometheus/2.43.0" 2024-03-07T03:43:43.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:43 smithi189 bash[66969]: cluster 2024-03-07T03:43:41.539239+0000 mgr.y (mgr.34107) 955 : cluster 0 pgmap v1546: 105 pgs: 105 active+clean; 483 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.7 MiB/s wr, 42 op/s 2024-03-07T03:43:43.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:43 smithi145 bash[79878]: cluster 2024-03-07T03:43:41.539239+0000 mgr.y (mgr.34107) 955 : cluster 0 pgmap v1546: 105 pgs: 105 active+clean; 483 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.7 MiB/s wr, 42 op/s 2024-03-07T03:43:43.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:43 smithi145 bash[82329]: cluster 2024-03-07T03:43:41.539239+0000 mgr.y (mgr.34107) 955 : cluster 0 pgmap v1546: 105 pgs: 105 active+clean; 483 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.7 MiB/s wr, 42 op/s 2024-03-07T03:43:44.553 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/6.MinOverlap (22101 ms) 2024-03-07T03:43:44.553 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/6.ZeroOverlap 2024-03-07T03:43:44.553 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:43:45.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:45 smithi189 bash[66969]: cluster 2024-03-07T03:43:43.539858+0000 mgr.y (mgr.34107) 956 : cluster 0 pgmap v1547: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 4.8 MiB/s wr, 37 op/s 2024-03-07T03:43:45.563 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:45 smithi145 bash[82329]: cluster 2024-03-07T03:43:43.539858+0000 mgr.y (mgr.34107) 956 : cluster 0 pgmap v1547: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 4.8 MiB/s wr, 37 op/s 2024-03-07T03:43:45.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:45 smithi145 bash[79878]: cluster 2024-03-07T03:43:43.539858+0000 mgr.y (mgr.34107) 956 : cluster 0 pgmap v1547: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 4.8 MiB/s wr, 37 op/s 2024-03-07T03:43:47.334 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:47 smithi189 bash[66969]: cluster 2024-03-07T03:43:45.540784+0000 mgr.y (mgr.34107) 957 : cluster 0 pgmap v1548: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 759 KiB/s rd, 4.0 MiB/s wr, 38 op/s 2024-03-07T03:43:47.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:47 smithi145 bash[79878]: cluster 2024-03-07T03:43:45.540784+0000 mgr.y (mgr.34107) 957 : cluster 0 pgmap v1548: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 759 KiB/s rd, 4.0 MiB/s wr, 38 op/s 2024-03-07T03:43:47.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:47 smithi145 bash[82329]: cluster 2024-03-07T03:43:45.540784+0000 mgr.y (mgr.34107) 957 : cluster 0 pgmap v1548: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 759 KiB/s rd, 4.0 MiB/s wr, 38 op/s 2024-03-07T03:43:48.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:48 smithi145 bash[79878]: audit 2024-03-07T03:43:47.673912+0000 mon.a (mon.0) 1747 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:48.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:48 smithi145 bash[82329]: audit 2024-03-07T03:43:47.673912+0000 mon.a (mon.0) 1747 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:48.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:48 smithi189 bash[66969]: audit 2024-03-07T03:43:47.673912+0000 mon.a (mon.0) 1747 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:49.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:49 smithi145 bash[79878]: cluster 2024-03-07T03:43:47.541622+0000 mgr.y (mgr.34107) 958 : cluster 0 pgmap v1549: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 34 op/s 2024-03-07T03:43:49.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:49 smithi145 bash[82329]: cluster 2024-03-07T03:43:47.541622+0000 mgr.y (mgr.34107) 958 : cluster 0 pgmap v1549: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 34 op/s 2024-03-07T03:43:49.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:49 smithi189 bash[66969]: cluster 2024-03-07T03:43:47.541622+0000 mgr.y (mgr.34107) 958 : cluster 0 pgmap v1549: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 34 op/s 2024-03-07T03:43:50.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:50 smithi145 bash[79878]: cluster 2024-03-07T03:43:49.542342+0000 mgr.y (mgr.34107) 959 : cluster 0 pgmap v1550: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:43:50.564 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:50 smithi145 bash[82329]: cluster 2024-03-07T03:43:49.542342+0000 mgr.y (mgr.34107) 959 : cluster 0 pgmap v1550: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:43:50.583 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:50 smithi189 bash[66969]: cluster 2024-03-07T03:43:49.542342+0000 mgr.y (mgr.34107) 959 : cluster 0 pgmap v1550: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:43:52.584 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:43:52 smithi189 bash[59814]: ts=2024-03-07T03:43:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:43:53.063 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:52 smithi145 bash[79878]: cluster 2024-03-07T03:43:51.542982+0000 mgr.y (mgr.34107) 960 : cluster 0 pgmap v1551: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.3 MiB/s wr, 19 op/s 2024-03-07T03:43:53.064 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:43:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:43:52] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.43.0" 2024-03-07T03:43:53.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:52 smithi145 bash[82329]: cluster 2024-03-07T03:43:51.542982+0000 mgr.y (mgr.34107) 960 : cluster 0 pgmap v1551: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.3 MiB/s wr, 19 op/s 2024-03-07T03:43:53.083 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:52 smithi189 bash[66969]: cluster 2024-03-07T03:43:51.542982+0000 mgr.y (mgr.34107) 960 : cluster 0 pgmap v1551: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.3 MiB/s wr, 19 op/s 2024-03-07T03:43:55.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:54 smithi145 bash[79878]: cluster 2024-03-07T03:43:53.543848+0000 mgr.y (mgr.34107) 961 : cluster 0 pgmap v1552: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 16 op/s 2024-03-07T03:43:55.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:54 smithi145 bash[79878]: cluster 2024-03-07T03:43:53.612665+0000 mon.a (mon.0) 1748 : cluster 0 osdmap e967: 8 total, 8 up, 8 in 2024-03-07T03:43:55.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:54 smithi145 bash[82329]: cluster 2024-03-07T03:43:53.543848+0000 mgr.y (mgr.34107) 961 : cluster 0 pgmap v1552: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 16 op/s 2024-03-07T03:43:55.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:54 smithi145 bash[82329]: cluster 2024-03-07T03:43:53.612665+0000 mon.a (mon.0) 1748 : cluster 0 osdmap e967: 8 total, 8 up, 8 in 2024-03-07T03:43:55.083 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:54 smithi189 bash[66969]: cluster 2024-03-07T03:43:53.543848+0000 mgr.y (mgr.34107) 961 : cluster 0 pgmap v1552: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 16 op/s 2024-03-07T03:43:55.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:54 smithi189 bash[66969]: cluster 2024-03-07T03:43:53.612665+0000 mon.a (mon.0) 1748 : cluster 0 osdmap e967: 8 total, 8 up, 8 in 2024-03-07T03:43:56.063 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:55 smithi145 bash[79878]: audit 2024-03-07T03:43:54.701142+0000 mon.a (mon.0) 1749 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:43:56.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:55 smithi145 bash[82329]: audit 2024-03-07T03:43:54.701142+0000 mon.a (mon.0) 1749 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:43:56.083 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:55 smithi189 bash[66969]: audit 2024-03-07T03:43:54.701142+0000 mon.a (mon.0) 1749 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:43:57.063 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:56 smithi145 bash[79878]: cluster 2024-03-07T03:43:55.544606+0000 mgr.y (mgr.34107) 962 : cluster 0 pgmap v1554: 105 pgs: 105 active+clean; 511 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.6 MiB/s wr, 34 op/s 2024-03-07T03:43:57.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:56 smithi145 bash[82329]: cluster 2024-03-07T03:43:55.544606+0000 mgr.y (mgr.34107) 962 : cluster 0 pgmap v1554: 105 pgs: 105 active+clean; 511 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.6 MiB/s wr, 34 op/s 2024-03-07T03:43:57.083 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:56 smithi189 bash[66969]: cluster 2024-03-07T03:43:55.544606+0000 mgr.y (mgr.34107) 962 : cluster 0 pgmap v1554: 105 pgs: 105 active+clean; 511 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.6 MiB/s wr, 34 op/s 2024-03-07T03:43:59.063 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:43:58 smithi145 bash[79878]: cluster 2024-03-07T03:43:57.545321+0000 mgr.y (mgr.34107) 963 : cluster 0 pgmap v1555: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-03-07T03:43:59.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:43:58 smithi145 bash[82329]: cluster 2024-03-07T03:43:57.545321+0000 mgr.y (mgr.34107) 963 : cluster 0 pgmap v1555: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-03-07T03:43:59.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:43:58 smithi189 bash[66969]: cluster 2024-03-07T03:43:57.545321+0000 mgr.y (mgr.34107) 963 : cluster 0 pgmap v1555: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-03-07T03:44:01.063 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:00 smithi145 bash[79878]: cluster 2024-03-07T03:43:59.545979+0000 mgr.y (mgr.34107) 964 : cluster 0 pgmap v1556: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:44:01.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:00 smithi145 bash[82329]: cluster 2024-03-07T03:43:59.545979+0000 mgr.y (mgr.34107) 964 : cluster 0 pgmap v1556: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:44:01.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:00 smithi189 bash[66969]: cluster 2024-03-07T03:43:59.545979+0000 mgr.y (mgr.34107) 964 : cluster 0 pgmap v1556: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:44:02.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[79878]: audit 2024-03-07T03:44:00.640314+0000 mon.a (mon.0) 1750 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[79878]: audit 2024-03-07T03:44:00.650107+0000 mon.a (mon.0) 1751 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[79878]: audit 2024-03-07T03:44:00.794908+0000 mon.a (mon.0) 1752 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[79878]: audit 2024-03-07T03:44:00.806023+0000 mon.a (mon.0) 1753 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[79878]: audit 2024-03-07T03:44:01.383330+0000 mon.a (mon.0) 1754 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:44:02.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[79878]: audit 2024-03-07T03:44:01.384190+0000 mon.a (mon.0) 1755 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:44:02.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[79878]: audit 2024-03-07T03:44:01.389280+0000 mon.a (mon.0) 1756 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[82329]: audit 2024-03-07T03:44:00.640314+0000 mon.a (mon.0) 1750 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[82329]: audit 2024-03-07T03:44:00.650107+0000 mon.a (mon.0) 1751 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[82329]: audit 2024-03-07T03:44:00.794908+0000 mon.a (mon.0) 1752 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[82329]: audit 2024-03-07T03:44:00.806023+0000 mon.a (mon.0) 1753 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[82329]: audit 2024-03-07T03:44:01.383330+0000 mon.a (mon.0) 1754 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:44:02.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[82329]: audit 2024-03-07T03:44:01.384190+0000 mon.a (mon.0) 1755 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:44:02.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:01 smithi145 bash[82329]: audit 2024-03-07T03:44:01.389280+0000 mon.a (mon.0) 1756 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:01 smithi189 bash[66969]: audit 2024-03-07T03:44:00.640314+0000 mon.a (mon.0) 1750 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:01 smithi189 bash[66969]: audit 2024-03-07T03:44:00.650107+0000 mon.a (mon.0) 1751 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:01 smithi189 bash[66969]: audit 2024-03-07T03:44:00.794908+0000 mon.a (mon.0) 1752 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:01 smithi189 bash[66969]: audit 2024-03-07T03:44:00.806023+0000 mon.a (mon.0) 1753 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:01 smithi189 bash[66969]: audit 2024-03-07T03:44:01.383330+0000 mon.a (mon.0) 1754 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:44:02.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:01 smithi189 bash[66969]: audit 2024-03-07T03:44:01.384190+0000 mon.a (mon.0) 1755 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:44:02.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:01 smithi189 bash[66969]: audit 2024-03-07T03:44:01.389280+0000 mon.a (mon.0) 1756 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:02.584 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:44:02 smithi189 bash[59814]: ts=2024-03-07T03:44:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:44:03.063 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:02 smithi145 bash[79878]: cluster 2024-03-07T03:44:01.546656+0000 mgr.y (mgr.34107) 965 : cluster 0 pgmap v1557: 105 pgs: 105 active+clean; 544 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 4.9 MiB/s wr, 42 op/s 2024-03-07T03:44:03.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:02 smithi145 bash[82329]: cluster 2024-03-07T03:44:01.546656+0000 mgr.y (mgr.34107) 965 : cluster 0 pgmap v1557: 105 pgs: 105 active+clean; 544 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 4.9 MiB/s wr, 42 op/s 2024-03-07T03:44:03.064 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:44:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:44:02] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.43.0" 2024-03-07T03:44:03.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:02 smithi189 bash[66969]: cluster 2024-03-07T03:44:01.546656+0000 mgr.y (mgr.34107) 965 : cluster 0 pgmap v1557: 105 pgs: 105 active+clean; 544 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 4.9 MiB/s wr, 42 op/s 2024-03-07T03:44:04.063 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:03 smithi145 bash[79878]: audit 2024-03-07T03:44:02.674161+0000 mon.a (mon.0) 1757 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:04.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:03 smithi145 bash[82329]: audit 2024-03-07T03:44:02.674161+0000 mon.a (mon.0) 1757 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:04.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:03 smithi189 bash[66969]: audit 2024-03-07T03:44:02.674161+0000 mon.a (mon.0) 1757 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:05.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:04 smithi145 bash[79878]: cluster 2024-03-07T03:44:03.547256+0000 mgr.y (mgr.34107) 966 : cluster 0 pgmap v1558: 105 pgs: 105 active+clean; 548 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 5.3 MiB/s wr, 43 op/s 2024-03-07T03:44:05.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:04 smithi145 bash[82329]: cluster 2024-03-07T03:44:03.547256+0000 mgr.y (mgr.34107) 966 : cluster 0 pgmap v1558: 105 pgs: 105 active+clean; 548 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 5.3 MiB/s wr, 43 op/s 2024-03-07T03:44:05.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:04 smithi189 bash[66969]: cluster 2024-03-07T03:44:03.547256+0000 mgr.y (mgr.34107) 966 : cluster 0 pgmap v1558: 105 pgs: 105 active+clean; 548 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 5.3 MiB/s wr, 43 op/s 2024-03-07T03:44:05.862 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/6.ZeroOverlap (21309 ms) 2024-03-07T03:44:06.680 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/6 (159949 ms total) 2024-03-07T03:44:06.680 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:44:06.680 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/7, where TypeParam = LUKS2UnderLUKS1 2024-03-07T03:44:06.727 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/7.Simple 2024-03-07T03:44:07.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:06 smithi145 bash[79878]: cluster 2024-03-07T03:44:05.547929+0000 mgr.y (mgr.34107) 967 : cluster 0 pgmap v1559: 105 pgs: 105 active+clean; 552 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 4.8 MiB/s wr, 39 op/s 2024-03-07T03:44:07.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:06 smithi145 bash[82329]: cluster 2024-03-07T03:44:05.547929+0000 mgr.y (mgr.34107) 967 : cluster 0 pgmap v1559: 105 pgs: 105 active+clean; 552 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 4.8 MiB/s wr, 39 op/s 2024-03-07T03:44:07.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:06 smithi189 bash[66969]: cluster 2024-03-07T03:44:05.547929+0000 mgr.y (mgr.34107) 967 : cluster 0 pgmap v1559: 105 pgs: 105 active+clean; 552 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 395 KiB/s rd, 4.8 MiB/s wr, 39 op/s 2024-03-07T03:44:08.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:07 smithi145 bash[79878]: cluster 2024-03-07T03:44:06.669009+0000 mon.a (mon.0) 1758 : cluster 0 osdmap e968: 8 total, 8 up, 8 in 2024-03-07T03:44:08.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:07 smithi145 bash[82329]: cluster 2024-03-07T03:44:06.669009+0000 mon.a (mon.0) 1758 : cluster 0 osdmap e968: 8 total, 8 up, 8 in 2024-03-07T03:44:08.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:07 smithi189 bash[66969]: cluster 2024-03-07T03:44:06.669009+0000 mon.a (mon.0) 1758 : cluster 0 osdmap e968: 8 total, 8 up, 8 in 2024-03-07T03:44:08.708 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:44:09.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:08 smithi145 bash[79878]: cluster 2024-03-07T03:44:07.548570+0000 mgr.y (mgr.34107) 968 : cluster 0 pgmap v1561: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:44:09.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:08 smithi145 bash[79878]: cluster 2024-03-07T03:44:07.704152+0000 mon.a (mon.0) 1759 : cluster 0 osdmap e969: 8 total, 8 up, 8 in 2024-03-07T03:44:09.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:08 smithi145 bash[79878]: audit 2024-03-07T03:44:07.705824+0000 mon.a (mon.0) 1760 : audit 1 from='client.? 172.21.15.145:0/22599586' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-43","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:44:09.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:08 smithi145 bash[82329]: cluster 2024-03-07T03:44:07.548570+0000 mgr.y (mgr.34107) 968 : cluster 0 pgmap v1561: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:44:09.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:08 smithi145 bash[82329]: cluster 2024-03-07T03:44:07.704152+0000 mon.a (mon.0) 1759 : cluster 0 osdmap e969: 8 total, 8 up, 8 in 2024-03-07T03:44:09.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:08 smithi145 bash[82329]: audit 2024-03-07T03:44:07.705824+0000 mon.a (mon.0) 1760 : audit 1 from='client.? 172.21.15.145:0/22599586' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-43","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:44:09.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:08 smithi189 bash[66969]: cluster 2024-03-07T03:44:07.548570+0000 mgr.y (mgr.34107) 968 : cluster 0 pgmap v1561: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:44:09.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:08 smithi189 bash[66969]: cluster 2024-03-07T03:44:07.704152+0000 mon.a (mon.0) 1759 : cluster 0 osdmap e969: 8 total, 8 up, 8 in 2024-03-07T03:44:09.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:08 smithi189 bash[66969]: audit 2024-03-07T03:44:07.705824+0000 mon.a (mon.0) 1760 : audit 1 from='client.? 172.21.15.145:0/22599586' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-43","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:44:10.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:09 smithi145 bash[79878]: audit 2024-03-07T03:44:08.692356+0000 mon.a (mon.0) 1761 : audit 1 from='client.? 172.21.15.145:0/22599586' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-43","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:44:10.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:09 smithi145 bash[79878]: cluster 2024-03-07T03:44:08.700187+0000 mon.a (mon.0) 1762 : cluster 0 osdmap e970: 8 total, 8 up, 8 in 2024-03-07T03:44:10.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:09 smithi145 bash[82329]: audit 2024-03-07T03:44:08.692356+0000 mon.a (mon.0) 1761 : audit 1 from='client.? 172.21.15.145:0/22599586' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-43","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:44:10.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:09 smithi145 bash[82329]: cluster 2024-03-07T03:44:08.700187+0000 mon.a (mon.0) 1762 : cluster 0 osdmap e970: 8 total, 8 up, 8 in 2024-03-07T03:44:10.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:09 smithi189 bash[66969]: audit 2024-03-07T03:44:08.692356+0000 mon.a (mon.0) 1761 : audit 1 from='client.? 172.21.15.145:0/22599586' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-43","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:44:10.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:09 smithi189 bash[66969]: cluster 2024-03-07T03:44:08.700187+0000 mon.a (mon.0) 1762 : cluster 0 osdmap e970: 8 total, 8 up, 8 in 2024-03-07T03:44:11.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:10 smithi145 bash[82329]: cluster 2024-03-07T03:44:09.549586+0000 mgr.y (mgr.34107) 969 : cluster 0 pgmap v1564: 105 pgs: 20 unknown, 85 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:44:11.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:10 smithi145 bash[82329]: cluster 2024-03-07T03:44:09.724782+0000 mon.a (mon.0) 1763 : cluster 0 osdmap e971: 8 total, 8 up, 8 in 2024-03-07T03:44:11.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:10 smithi145 bash[79878]: cluster 2024-03-07T03:44:09.549586+0000 mgr.y (mgr.34107) 969 : cluster 0 pgmap v1564: 105 pgs: 20 unknown, 85 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:44:11.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:10 smithi145 bash[79878]: cluster 2024-03-07T03:44:09.724782+0000 mon.a (mon.0) 1763 : cluster 0 osdmap e971: 8 total, 8 up, 8 in 2024-03-07T03:44:11.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:10 smithi189 bash[66969]: cluster 2024-03-07T03:44:09.549586+0000 mgr.y (mgr.34107) 969 : cluster 0 pgmap v1564: 105 pgs: 20 unknown, 85 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:44:11.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:10 smithi189 bash[66969]: cluster 2024-03-07T03:44:09.724782+0000 mon.a (mon.0) 1763 : cluster 0 osdmap e971: 8 total, 8 up, 8 in 2024-03-07T03:44:12.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:11 smithi145 bash[79878]: cluster 2024-03-07T03:44:10.734214+0000 mon.a (mon.0) 1764 : cluster 0 osdmap e972: 8 total, 8 up, 8 in 2024-03-07T03:44:12.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:11 smithi145 bash[82329]: cluster 2024-03-07T03:44:10.734214+0000 mon.a (mon.0) 1764 : cluster 0 osdmap e972: 8 total, 8 up, 8 in 2024-03-07T03:44:12.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:11 smithi189 bash[66969]: cluster 2024-03-07T03:44:10.734214+0000 mon.a (mon.0) 1764 : cluster 0 osdmap e972: 8 total, 8 up, 8 in 2024-03-07T03:44:12.584 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:44:12 smithi189 bash[59814]: ts=2024-03-07T03:44:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:44:13.064 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:44:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:44:12] "GET /metrics HTTP/1.1" 200 36588 "" "Prometheus/2.43.0" 2024-03-07T03:44:13.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:12 smithi145 bash[82329]: cluster 2024-03-07T03:44:11.550399+0000 mgr.y (mgr.34107) 970 : cluster 0 pgmap v1567: 105 pgs: 11 unknown, 94 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:44:13.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:12 smithi145 bash[79878]: cluster 2024-03-07T03:44:11.550399+0000 mgr.y (mgr.34107) 970 : cluster 0 pgmap v1567: 105 pgs: 11 unknown, 94 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:44:13.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:12 smithi189 bash[66969]: cluster 2024-03-07T03:44:11.550399+0000 mgr.y (mgr.34107) 970 : cluster 0 pgmap v1567: 105 pgs: 11 unknown, 94 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:44:15.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:14 smithi145 bash[79878]: cluster 2024-03-07T03:44:13.551065+0000 mgr.y (mgr.34107) 971 : cluster 0 pgmap v1568: 105 pgs: 105 active+clean; 582 KiB data, 614 MiB used, 715 GiB / 715 GiB avail; 524 B/s rd, 349 B/s wr, 1 op/s 2024-03-07T03:44:15.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:14 smithi145 bash[82329]: cluster 2024-03-07T03:44:13.551065+0000 mgr.y (mgr.34107) 971 : cluster 0 pgmap v1568: 105 pgs: 105 active+clean; 582 KiB data, 614 MiB used, 715 GiB / 715 GiB avail; 524 B/s rd, 349 B/s wr, 1 op/s 2024-03-07T03:44:15.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:14 smithi189 bash[66969]: cluster 2024-03-07T03:44:13.551065+0000 mgr.y (mgr.34107) 971 : cluster 0 pgmap v1568: 105 pgs: 105 active+clean; 582 KiB data, 614 MiB used, 715 GiB / 715 GiB avail; 524 B/s rd, 349 B/s wr, 1 op/s 2024-03-07T03:44:17.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:16 smithi145 bash[79878]: cluster 2024-03-07T03:44:15.551854+0000 mgr.y (mgr.34107) 972 : cluster 0 pgmap v1569: 105 pgs: 105 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 895 B/s wr, 6 op/s 2024-03-07T03:44:17.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:16 smithi145 bash[82329]: cluster 2024-03-07T03:44:15.551854+0000 mgr.y (mgr.34107) 972 : cluster 0 pgmap v1569: 105 pgs: 105 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 895 B/s wr, 6 op/s 2024-03-07T03:44:17.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:16 smithi189 bash[66969]: cluster 2024-03-07T03:44:15.551854+0000 mgr.y (mgr.34107) 972 : cluster 0 pgmap v1569: 105 pgs: 105 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 895 B/s wr, 6 op/s 2024-03-07T03:44:18.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:17 smithi145 bash[79878]: audit 2024-03-07T03:44:17.676558+0000 mon.a (mon.0) 1765 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:18.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:17 smithi145 bash[82329]: audit 2024-03-07T03:44:17.676558+0000 mon.a (mon.0) 1765 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:18.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:17 smithi189 bash[66969]: audit 2024-03-07T03:44:17.676558+0000 mon.a (mon.0) 1765 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:19.314 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:19 smithi145 bash[79878]: cluster 2024-03-07T03:44:17.552568+0000 mgr.y (mgr.34107) 973 : cluster 0 pgmap v1570: 105 pgs: 105 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-03-07T03:44:19.314 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:19 smithi145 bash[79878]: cluster 2024-03-07T03:44:18.061898+0000 mon.a (mon.0) 1766 : cluster 0 osdmap e973: 8 total, 8 up, 8 in 2024-03-07T03:44:19.314 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:19 smithi145 bash[82329]: cluster 2024-03-07T03:44:17.552568+0000 mgr.y (mgr.34107) 973 : cluster 0 pgmap v1570: 105 pgs: 105 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-03-07T03:44:19.314 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:19 smithi145 bash[82329]: cluster 2024-03-07T03:44:18.061898+0000 mon.a (mon.0) 1766 : cluster 0 osdmap e973: 8 total, 8 up, 8 in 2024-03-07T03:44:19.334 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:19 smithi189 bash[66969]: cluster 2024-03-07T03:44:17.552568+0000 mgr.y (mgr.34107) 973 : cluster 0 pgmap v1570: 105 pgs: 105 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-03-07T03:44:19.334 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:19 smithi189 bash[66969]: cluster 2024-03-07T03:44:18.061898+0000 mon.a (mon.0) 1766 : cluster 0 osdmap e973: 8 total, 8 up, 8 in 2024-03-07T03:44:21.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:20 smithi145 bash[82329]: cluster 2024-03-07T03:44:19.553128+0000 mgr.y (mgr.34107) 974 : cluster 0 pgmap v1572: 105 pgs: 105 active+clean; 8.6 MiB data, 666 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 929 KiB/s wr, 8 op/s 2024-03-07T03:44:21.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:20 smithi145 bash[82329]: audit 2024-03-07T03:44:19.702950+0000 mon.a (mon.0) 1767 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:21.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:20 smithi145 bash[79878]: cluster 2024-03-07T03:44:19.553128+0000 mgr.y (mgr.34107) 974 : cluster 0 pgmap v1572: 105 pgs: 105 active+clean; 8.6 MiB data, 666 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 929 KiB/s wr, 8 op/s 2024-03-07T03:44:21.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:20 smithi145 bash[79878]: audit 2024-03-07T03:44:19.702950+0000 mon.a (mon.0) 1767 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:21.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:20 smithi189 bash[66969]: cluster 2024-03-07T03:44:19.553128+0000 mgr.y (mgr.34107) 974 : cluster 0 pgmap v1572: 105 pgs: 105 active+clean; 8.6 MiB data, 666 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 929 KiB/s wr, 8 op/s 2024-03-07T03:44:21.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:20 smithi189 bash[66969]: audit 2024-03-07T03:44:19.702950+0000 mon.a (mon.0) 1767 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:44:22.584 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:44:22 smithi189 bash[59814]: ts=2024-03-07T03:44:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:44:23.064 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:44:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:44:22] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2024-03-07T03:44:23.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:22 smithi145 bash[79878]: cluster 2024-03-07T03:44:21.553663+0000 mgr.y (mgr.34107) 975 : cluster 0 pgmap v1573: 105 pgs: 105 active+clean; 29 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 2.8 MiB/s wr, 38 op/s 2024-03-07T03:44:23.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:22 smithi145 bash[82329]: cluster 2024-03-07T03:44:21.553663+0000 mgr.y (mgr.34107) 975 : cluster 0 pgmap v1573: 105 pgs: 105 active+clean; 29 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 2.8 MiB/s wr, 38 op/s 2024-03-07T03:44:23.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:22 smithi189 bash[66969]: cluster 2024-03-07T03:44:21.553663+0000 mgr.y (mgr.34107) 975 : cluster 0 pgmap v1573: 105 pgs: 105 active+clean; 29 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 2.8 MiB/s wr, 38 op/s 2024-03-07T03:44:25.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:24 smithi145 bash[79878]: cluster 2024-03-07T03:44:23.554202+0000 mgr.y (mgr.34107) 976 : cluster 0 pgmap v1574: 105 pgs: 105 active+clean; 42 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:44:25.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:24 smithi145 bash[82329]: cluster 2024-03-07T03:44:23.554202+0000 mgr.y (mgr.34107) 976 : cluster 0 pgmap v1574: 105 pgs: 105 active+clean; 42 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:44:25.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:24 smithi189 bash[66969]: cluster 2024-03-07T03:44:23.554202+0000 mgr.y (mgr.34107) 976 : cluster 0 pgmap v1574: 105 pgs: 105 active+clean; 42 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:44:27.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:26 smithi145 bash[82329]: cluster 2024-03-07T03:44:25.554893+0000 mgr.y (mgr.34107) 977 : cluster 0 pgmap v1575: 105 pgs: 105 active+clean; 42 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:44:27.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:26 smithi145 bash[79878]: cluster 2024-03-07T03:44:25.554893+0000 mgr.y (mgr.34107) 977 : cluster 0 pgmap v1575: 105 pgs: 105 active+clean; 42 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:44:27.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:26 smithi189 bash[66969]: cluster 2024-03-07T03:44:25.554893+0000 mgr.y (mgr.34107) 977 : cluster 0 pgmap v1575: 105 pgs: 105 active+clean; 42 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:44:29.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:28 smithi145 bash[79878]: cluster 2024-03-07T03:44:27.555492+0000 mgr.y (mgr.34107) 978 : cluster 0 pgmap v1576: 105 pgs: 105 active+clean; 46 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 216 KiB/s rd, 4.5 MiB/s wr, 39 op/s 2024-03-07T03:44:29.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:28 smithi145 bash[82329]: cluster 2024-03-07T03:44:27.555492+0000 mgr.y (mgr.34107) 978 : cluster 0 pgmap v1576: 105 pgs: 105 active+clean; 46 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 216 KiB/s rd, 4.5 MiB/s wr, 39 op/s 2024-03-07T03:44:29.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:28 smithi189 bash[66969]: cluster 2024-03-07T03:44:27.555492+0000 mgr.y (mgr.34107) 978 : cluster 0 pgmap v1576: 105 pgs: 105 active+clean; 46 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 216 KiB/s rd, 4.5 MiB/s wr, 39 op/s 2024-03-07T03:44:31.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:30 smithi145 bash[79878]: cluster 2024-03-07T03:44:29.556126+0000 mgr.y (mgr.34107) 979 : cluster 0 pgmap v1577: 105 pgs: 105 active+clean; 50 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 188 KiB/s rd, 4.3 MiB/s wr, 34 op/s 2024-03-07T03:44:31.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:30 smithi145 bash[82329]: cluster 2024-03-07T03:44:29.556126+0000 mgr.y (mgr.34107) 979 : cluster 0 pgmap v1577: 105 pgs: 105 active+clean; 50 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 188 KiB/s rd, 4.3 MiB/s wr, 34 op/s 2024-03-07T03:44:31.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:30 smithi189 bash[66969]: cluster 2024-03-07T03:44:29.556126+0000 mgr.y (mgr.34107) 979 : cluster 0 pgmap v1577: 105 pgs: 105 active+clean; 50 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 188 KiB/s rd, 4.3 MiB/s wr, 34 op/s 2024-03-07T03:44:32.281 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/7.Simple (25552 ms) 2024-03-07T03:44:32.281 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/7.Grow 2024-03-07T03:44:32.281 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:44:32.584 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:44:32 smithi189 bash[59814]: ts=2024-03-07T03:44:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:44:33.064 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:44:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:44:32] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2024-03-07T03:44:33.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:32 smithi145 bash[82329]: cluster 2024-03-07T03:44:31.556990+0000 mgr.y (mgr.34107) 980 : cluster 0 pgmap v1578: 105 pgs: 105 active+clean; 50 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.4 MiB/s wr, 41 op/s 2024-03-07T03:44:33.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:32 smithi145 bash[82329]: audit 2024-03-07T03:44:32.675561+0000 mon.a (mon.0) 1768 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:33.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:32 smithi145 bash[79878]: cluster 2024-03-07T03:44:31.556990+0000 mgr.y (mgr.34107) 980 : cluster 0 pgmap v1578: 105 pgs: 105 active+clean; 50 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.4 MiB/s wr, 41 op/s 2024-03-07T03:44:33.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:32 smithi145 bash[79878]: audit 2024-03-07T03:44:32.675561+0000 mon.a (mon.0) 1768 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:33.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:32 smithi189 bash[66969]: cluster 2024-03-07T03:44:31.556990+0000 mgr.y (mgr.34107) 980 : cluster 0 pgmap v1578: 105 pgs: 105 active+clean; 50 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.4 MiB/s wr, 41 op/s 2024-03-07T03:44:33.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:32 smithi189 bash[66969]: audit 2024-03-07T03:44:32.675561+0000 mon.a (mon.0) 1768 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:35.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:34 smithi145 bash[79878]: cluster 2024-03-07T03:44:33.557690+0000 mgr.y (mgr.34107) 981 : cluster 0 pgmap v1579: 105 pgs: 105 active+clean; 54 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2024-03-07T03:44:35.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:34 smithi145 bash[82329]: cluster 2024-03-07T03:44:33.557690+0000 mgr.y (mgr.34107) 981 : cluster 0 pgmap v1579: 105 pgs: 105 active+clean; 54 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2024-03-07T03:44:35.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:34 smithi189 bash[66969]: cluster 2024-03-07T03:44:33.557690+0000 mgr.y (mgr.34107) 981 : cluster 0 pgmap v1579: 105 pgs: 105 active+clean; 54 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2024-03-07T03:44:37.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:36 smithi145 bash[79878]: cluster 2024-03-07T03:44:35.558412+0000 mgr.y (mgr.34107) 982 : cluster 0 pgmap v1580: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 3.0 MiB/s wr, 26 op/s 2024-03-07T03:44:37.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:36 smithi145 bash[82329]: cluster 2024-03-07T03:44:35.558412+0000 mgr.y (mgr.34107) 982 : cluster 0 pgmap v1580: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 3.0 MiB/s wr, 26 op/s 2024-03-07T03:44:37.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:36 smithi189 bash[66969]: cluster 2024-03-07T03:44:35.558412+0000 mgr.y (mgr.34107) 982 : cluster 0 pgmap v1580: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 3.0 MiB/s wr, 26 op/s 2024-03-07T03:44:39.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:38 smithi145 bash[79878]: cluster 2024-03-07T03:44:37.559424+0000 mgr.y (mgr.34107) 983 : cluster 0 pgmap v1581: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 26 op/s 2024-03-07T03:44:39.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:38 smithi145 bash[82329]: cluster 2024-03-07T03:44:37.559424+0000 mgr.y (mgr.34107) 983 : cluster 0 pgmap v1581: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 26 op/s 2024-03-07T03:44:39.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:38 smithi189 bash[66969]: cluster 2024-03-07T03:44:37.559424+0000 mgr.y (mgr.34107) 983 : cluster 0 pgmap v1581: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 26 op/s 2024-03-07T03:44:41.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:40 smithi145 bash[79878]: cluster 2024-03-07T03:44:39.560372+0000 mgr.y (mgr.34107) 984 : cluster 0 pgmap v1582: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-03-07T03:44:41.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:40 smithi145 bash[82329]: cluster 2024-03-07T03:44:39.560372+0000 mgr.y (mgr.34107) 984 : cluster 0 pgmap v1582: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-03-07T03:44:41.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:40 smithi189 bash[66969]: cluster 2024-03-07T03:44:39.560372+0000 mgr.y (mgr.34107) 984 : cluster 0 pgmap v1582: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.7 MiB/s wr, 23 op/s 2024-03-07T03:44:42.584 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:44:42 smithi189 bash[59814]: ts=2024-03-07T03:44:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:44:43.064 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:44:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:44:42] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2024-03-07T03:44:43.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:42 smithi145 bash[79878]: cluster 2024-03-07T03:44:41.561376+0000 mgr.y (mgr.34107) 985 : cluster 0 pgmap v1583: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.3 MiB/s wr, 23 op/s 2024-03-07T03:44:43.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:42 smithi145 bash[82329]: cluster 2024-03-07T03:44:41.561376+0000 mgr.y (mgr.34107) 985 : cluster 0 pgmap v1583: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.3 MiB/s wr, 23 op/s 2024-03-07T03:44:43.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:42 smithi189 bash[66969]: cluster 2024-03-07T03:44:41.561376+0000 mgr.y (mgr.34107) 985 : cluster 0 pgmap v1583: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.3 MiB/s wr, 23 op/s 2024-03-07T03:44:44.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:43 smithi145 bash[79878]: cluster 2024-03-07T03:44:42.759760+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e974: 8 total, 8 up, 8 in 2024-03-07T03:44:44.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:43 smithi145 bash[82329]: cluster 2024-03-07T03:44:42.759760+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e974: 8 total, 8 up, 8 in 2024-03-07T03:44:44.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:43 smithi189 bash[66969]: cluster 2024-03-07T03:44:42.759760+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e974: 8 total, 8 up, 8 in 2024-03-07T03:44:45.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:44 smithi145 bash[79878]: cluster 2024-03-07T03:44:43.562012+0000 mgr.y (mgr.34107) 986 : cluster 0 pgmap v1585: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.0 MiB/s wr, 8 op/s 2024-03-07T03:44:45.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:44 smithi145 bash[82329]: cluster 2024-03-07T03:44:43.562012+0000 mgr.y (mgr.34107) 986 : cluster 0 pgmap v1585: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.0 MiB/s wr, 8 op/s 2024-03-07T03:44:45.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:44 smithi189 bash[66969]: cluster 2024-03-07T03:44:43.562012+0000 mgr.y (mgr.34107) 986 : cluster 0 pgmap v1585: 105 pgs: 105 active+clean; 74 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.0 MiB/s wr, 8 op/s 2024-03-07T03:44:47.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:46 smithi145 bash[82329]: cluster 2024-03-07T03:44:45.563025+0000 mgr.y (mgr.34107) 987 : cluster 0 pgmap v1586: 105 pgs: 105 active+clean; 111 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 435 KiB/s rd, 3.7 MiB/s wr, 35 op/s 2024-03-07T03:44:47.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:46 smithi145 bash[79878]: cluster 2024-03-07T03:44:45.563025+0000 mgr.y (mgr.34107) 987 : cluster 0 pgmap v1586: 105 pgs: 105 active+clean; 111 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 435 KiB/s rd, 3.7 MiB/s wr, 35 op/s 2024-03-07T03:44:47.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:46 smithi189 bash[66969]: cluster 2024-03-07T03:44:45.563025+0000 mgr.y (mgr.34107) 987 : cluster 0 pgmap v1586: 105 pgs: 105 active+clean; 111 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 435 KiB/s rd, 3.7 MiB/s wr, 35 op/s 2024-03-07T03:44:48.064 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:47 smithi145 bash[82329]: audit 2024-03-07T03:44:47.675740+0000 mon.a (mon.0) 1770 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:48.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:47 smithi145 bash[79878]: audit 2024-03-07T03:44:47.675740+0000 mon.a (mon.0) 1770 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:48.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:47 smithi189 bash[66969]: audit 2024-03-07T03:44:47.675740+0000 mon.a (mon.0) 1770 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:49.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:48 smithi145 bash[79878]: cluster 2024-03-07T03:44:47.563610+0000 mgr.y (mgr.34107) 988 : cluster 0 pgmap v1587: 105 pgs: 105 active+clean; 115 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:44:49.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:48 smithi145 bash[82329]: cluster 2024-03-07T03:44:47.563610+0000 mgr.y (mgr.34107) 988 : cluster 0 pgmap v1587: 105 pgs: 105 active+clean; 115 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:44:49.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:48 smithi189 bash[66969]: cluster 2024-03-07T03:44:47.563610+0000 mgr.y (mgr.34107) 988 : cluster 0 pgmap v1587: 105 pgs: 105 active+clean; 115 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:44:51.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:50 smithi145 bash[79878]: cluster 2024-03-07T03:44:49.564254+0000 mgr.y (mgr.34107) 989 : cluster 0 pgmap v1588: 105 pgs: 105 active+clean; 115 MiB data, 968 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:44:51.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:50 smithi145 bash[82329]: cluster 2024-03-07T03:44:49.564254+0000 mgr.y (mgr.34107) 989 : cluster 0 pgmap v1588: 105 pgs: 105 active+clean; 115 MiB data, 968 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:44:51.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:50 smithi189 bash[66969]: cluster 2024-03-07T03:44:49.564254+0000 mgr.y (mgr.34107) 989 : cluster 0 pgmap v1588: 105 pgs: 105 active+clean; 115 MiB data, 968 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:44:52.585 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:44:52 smithi189 bash[59814]: ts=2024-03-07T03:44:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:44:53.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:52 smithi145 bash[79878]: cluster 2024-03-07T03:44:51.564834+0000 mgr.y (mgr.34107) 990 : cluster 0 pgmap v1589: 105 pgs: 105 active+clean; 119 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-03-07T03:44:53.065 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:44:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:44:52] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.43.0" 2024-03-07T03:44:53.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:52 smithi145 bash[82329]: cluster 2024-03-07T03:44:51.564834+0000 mgr.y (mgr.34107) 990 : cluster 0 pgmap v1589: 105 pgs: 105 active+clean; 119 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-03-07T03:44:53.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:52 smithi189 bash[66969]: cluster 2024-03-07T03:44:51.564834+0000 mgr.y (mgr.34107) 990 : cluster 0 pgmap v1589: 105 pgs: 105 active+clean; 119 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-03-07T03:44:55.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:54 smithi145 bash[79878]: cluster 2024-03-07T03:44:53.565395+0000 mgr.y (mgr.34107) 991 : cluster 0 pgmap v1590: 105 pgs: 105 active+clean; 119 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 4.2 MiB/s wr, 35 op/s 2024-03-07T03:44:55.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:54 smithi145 bash[82329]: cluster 2024-03-07T03:44:53.565395+0000 mgr.y (mgr.34107) 991 : cluster 0 pgmap v1590: 105 pgs: 105 active+clean; 119 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 4.2 MiB/s wr, 35 op/s 2024-03-07T03:44:55.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:54 smithi189 bash[66969]: cluster 2024-03-07T03:44:53.565395+0000 mgr.y (mgr.34107) 991 : cluster 0 pgmap v1590: 105 pgs: 105 active+clean; 119 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 4.2 MiB/s wr, 35 op/s 2024-03-07T03:44:57.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:56 smithi145 bash[79878]: cluster 2024-03-07T03:44:55.567568+0000 mgr.y (mgr.34107) 992 : cluster 0 pgmap v1591: 105 pgs: 105 active+clean; 119 MiB data, 980 MiB used, 714 GiB / 715 GiB avail; 209 KiB/s rd, 3.8 MiB/s wr, 38 op/s 2024-03-07T03:44:57.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:56 smithi145 bash[82329]: cluster 2024-03-07T03:44:55.567568+0000 mgr.y (mgr.34107) 992 : cluster 0 pgmap v1591: 105 pgs: 105 active+clean; 119 MiB data, 980 MiB used, 714 GiB / 715 GiB avail; 209 KiB/s rd, 3.8 MiB/s wr, 38 op/s 2024-03-07T03:44:57.084 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:56 smithi189 bash[66969]: cluster 2024-03-07T03:44:55.567568+0000 mgr.y (mgr.34107) 992 : cluster 0 pgmap v1591: 105 pgs: 105 active+clean; 119 MiB data, 980 MiB used, 714 GiB / 715 GiB avail; 209 KiB/s rd, 3.8 MiB/s wr, 38 op/s 2024-03-07T03:44:57.833 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/7.Grow (25553 ms) 2024-03-07T03:44:57.834 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/7.Shrink 2024-03-07T03:44:57.834 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:44:59.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:44:58 smithi145 bash[79878]: cluster 2024-03-07T03:44:57.568238+0000 mgr.y (mgr.34107) 993 : cluster 0 pgmap v1592: 105 pgs: 105 active+clean; 143 MiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.0 MiB/s wr, 17 op/s 2024-03-07T03:44:59.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:44:58 smithi145 bash[82329]: cluster 2024-03-07T03:44:57.568238+0000 mgr.y (mgr.34107) 993 : cluster 0 pgmap v1592: 105 pgs: 105 active+clean; 143 MiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.0 MiB/s wr, 17 op/s 2024-03-07T03:44:59.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:44:58 smithi189 bash[66969]: cluster 2024-03-07T03:44:57.568238+0000 mgr.y (mgr.34107) 993 : cluster 0 pgmap v1592: 105 pgs: 105 active+clean; 143 MiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.0 MiB/s wr, 17 op/s 2024-03-07T03:45:01.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:00 smithi145 bash[79878]: cluster 2024-03-07T03:44:59.568963+0000 mgr.y (mgr.34107) 994 : cluster 0 pgmap v1593: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.0 MiB/s wr, 25 op/s 2024-03-07T03:45:01.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:00 smithi145 bash[82329]: cluster 2024-03-07T03:44:59.568963+0000 mgr.y (mgr.34107) 994 : cluster 0 pgmap v1593: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.0 MiB/s wr, 25 op/s 2024-03-07T03:45:01.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:00 smithi189 bash[66969]: cluster 2024-03-07T03:44:59.568963+0000 mgr.y (mgr.34107) 994 : cluster 0 pgmap v1593: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 3.0 MiB/s wr, 25 op/s 2024-03-07T03:45:02.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:01 smithi145 bash[79878]: audit 2024-03-07T03:45:01.467115+0000 mon.a (mon.0) 1771 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:45:02.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:01 smithi145 bash[82329]: audit 2024-03-07T03:45:01.467115+0000 mon.a (mon.0) 1771 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:45:02.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:01 smithi189 bash[66969]: audit 2024-03-07T03:45:01.467115+0000 mon.a (mon.0) 1771 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:45:02.585 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:45:02 smithi189 bash[59814]: ts=2024-03-07T03:45:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:45:03.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:02 smithi145 bash[79878]: cluster 2024-03-07T03:45:01.569691+0000 mgr.y (mgr.34107) 995 : cluster 0 pgmap v1594: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-03-07T03:45:03.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:02 smithi145 bash[79878]: audit 2024-03-07T03:45:02.675576+0000 mon.a (mon.0) 1772 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:03.065 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:45:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:45:02] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.43.0" 2024-03-07T03:45:03.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:02 smithi145 bash[82329]: cluster 2024-03-07T03:45:01.569691+0000 mgr.y (mgr.34107) 995 : cluster 0 pgmap v1594: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-03-07T03:45:03.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:02 smithi145 bash[82329]: audit 2024-03-07T03:45:02.675576+0000 mon.a (mon.0) 1772 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:03.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:02 smithi189 bash[66969]: cluster 2024-03-07T03:45:01.569691+0000 mgr.y (mgr.34107) 995 : cluster 0 pgmap v1594: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-03-07T03:45:03.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:02 smithi189 bash[66969]: audit 2024-03-07T03:45:02.675576+0000 mon.a (mon.0) 1772 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:05.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:04 smithi189 bash[66969]: cluster 2024-03-07T03:45:03.570439+0000 mgr.y (mgr.34107) 996 : cluster 0 pgmap v1595: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 26 op/s 2024-03-07T03:45:05.315 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:04 smithi145 bash[79878]: cluster 2024-03-07T03:45:03.570439+0000 mgr.y (mgr.34107) 996 : cluster 0 pgmap v1595: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 26 op/s 2024-03-07T03:45:05.315 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:04 smithi145 bash[82329]: cluster 2024-03-07T03:45:03.570439+0000 mgr.y (mgr.34107) 996 : cluster 0 pgmap v1595: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 26 op/s 2024-03-07T03:45:07.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:06 smithi189 bash[66969]: cluster 2024-03-07T03:45:05.571249+0000 mgr.y (mgr.34107) 997 : cluster 0 pgmap v1596: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 27 op/s 2024-03-07T03:45:07.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:06 smithi189 bash[66969]: cluster 2024-03-07T03:45:05.841832+0000 mon.a (mon.0) 1773 : cluster 0 osdmap e975: 8 total, 8 up, 8 in 2024-03-07T03:45:07.315 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:06 smithi145 bash[82329]: cluster 2024-03-07T03:45:05.571249+0000 mgr.y (mgr.34107) 997 : cluster 0 pgmap v1596: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 27 op/s 2024-03-07T03:45:07.315 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:06 smithi145 bash[82329]: cluster 2024-03-07T03:45:05.841832+0000 mon.a (mon.0) 1773 : cluster 0 osdmap e975: 8 total, 8 up, 8 in 2024-03-07T03:45:07.315 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:06 smithi145 bash[79878]: cluster 2024-03-07T03:45:05.571249+0000 mgr.y (mgr.34107) 997 : cluster 0 pgmap v1596: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 27 op/s 2024-03-07T03:45:07.315 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:06 smithi145 bash[79878]: cluster 2024-03-07T03:45:05.841832+0000 mon.a (mon.0) 1773 : cluster 0 osdmap e975: 8 total, 8 up, 8 in 2024-03-07T03:45:08.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[79878]: audit 2024-03-07T03:45:07.303495+0000 mon.a (mon.0) 1774 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[79878]: audit 2024-03-07T03:45:07.315514+0000 mon.a (mon.0) 1775 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.565 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[79878]: audit 2024-03-07T03:45:07.504770+0000 mon.a (mon.0) 1776 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[79878]: audit 2024-03-07T03:45:07.517656+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[79878]: cluster 2024-03-07T03:45:07.571853+0000 mgr.y (mgr.34107) 998 : cluster 0 pgmap v1598: 105 pgs: 105 active+clean; 176 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 3.3 MiB/s wr, 18 op/s 2024-03-07T03:45:08.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[79878]: audit 2024-03-07T03:45:07.738560+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.16", "id": [1, 2]}]: dispatch 2024-03-07T03:45:08.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[79878]: audit 2024-03-07T03:45:08.155067+0000 mon.a (mon.0) 1779 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:45:08.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[79878]: audit 2024-03-07T03:45:08.156457+0000 mon.a (mon.0) 1780 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:45:08.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[79878]: audit 2024-03-07T03:45:08.164976+0000 mon.a (mon.0) 1781 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[82329]: audit 2024-03-07T03:45:07.303495+0000 mon.a (mon.0) 1774 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[82329]: audit 2024-03-07T03:45:07.315514+0000 mon.a (mon.0) 1775 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[82329]: audit 2024-03-07T03:45:07.504770+0000 mon.a (mon.0) 1776 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[82329]: audit 2024-03-07T03:45:07.517656+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[82329]: cluster 2024-03-07T03:45:07.571853+0000 mgr.y (mgr.34107) 998 : cluster 0 pgmap v1598: 105 pgs: 105 active+clean; 176 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 3.3 MiB/s wr, 18 op/s 2024-03-07T03:45:08.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[82329]: audit 2024-03-07T03:45:07.738560+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.16", "id": [1, 2]}]: dispatch 2024-03-07T03:45:08.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[82329]: audit 2024-03-07T03:45:08.155067+0000 mon.a (mon.0) 1779 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:45:08.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[82329]: audit 2024-03-07T03:45:08.156457+0000 mon.a (mon.0) 1780 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:45:08.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:08 smithi145 bash[82329]: audit 2024-03-07T03:45:08.164976+0000 mon.a (mon.0) 1781 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:08 smithi189 bash[66969]: audit 2024-03-07T03:45:07.303495+0000 mon.a (mon.0) 1774 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:08 smithi189 bash[66969]: audit 2024-03-07T03:45:07.315514+0000 mon.a (mon.0) 1775 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:08 smithi189 bash[66969]: audit 2024-03-07T03:45:07.504770+0000 mon.a (mon.0) 1776 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:08 smithi189 bash[66969]: audit 2024-03-07T03:45:07.517656+0000 mon.a (mon.0) 1777 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:08.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:08 smithi189 bash[66969]: cluster 2024-03-07T03:45:07.571853+0000 mgr.y (mgr.34107) 998 : cluster 0 pgmap v1598: 105 pgs: 105 active+clean; 176 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 3.3 MiB/s wr, 18 op/s 2024-03-07T03:45:08.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:08 smithi189 bash[66969]: audit 2024-03-07T03:45:07.738560+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.16", "id": [1, 2]}]: dispatch 2024-03-07T03:45:08.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:08 smithi189 bash[66969]: audit 2024-03-07T03:45:08.155067+0000 mon.a (mon.0) 1779 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:45:08.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:08 smithi189 bash[66969]: audit 2024-03-07T03:45:08.156457+0000 mon.a (mon.0) 1780 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:45:08.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:08 smithi189 bash[66969]: audit 2024-03-07T03:45:08.164976+0000 mon.a (mon.0) 1781 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:45:09.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:09 smithi145 bash[82329]: audit 2024-03-07T03:45:08.521694+0000 mon.a (mon.0) 1782 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.16", "id": [1, 2]}]': finished 2024-03-07T03:45:09.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:09 smithi145 bash[82329]: cluster 2024-03-07T03:45:08.526588+0000 mon.a (mon.0) 1783 : cluster 0 osdmap e976: 8 total, 8 up, 8 in 2024-03-07T03:45:09.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:09 smithi145 bash[79878]: audit 2024-03-07T03:45:08.521694+0000 mon.a (mon.0) 1782 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.16", "id": [1, 2]}]': finished 2024-03-07T03:45:09.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:09 smithi145 bash[79878]: cluster 2024-03-07T03:45:08.526588+0000 mon.a (mon.0) 1783 : cluster 0 osdmap e976: 8 total, 8 up, 8 in 2024-03-07T03:45:09.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:09 smithi189 bash[66969]: audit 2024-03-07T03:45:08.521694+0000 mon.a (mon.0) 1782 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.16", "id": [1, 2]}]': finished 2024-03-07T03:45:09.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:09 smithi189 bash[66969]: cluster 2024-03-07T03:45:08.526588+0000 mon.a (mon.0) 1783 : cluster 0 osdmap e976: 8 total, 8 up, 8 in 2024-03-07T03:45:10.814 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:10 smithi145 bash[79878]: cluster 2024-03-07T03:45:09.537830+0000 mon.a (mon.0) 1784 : cluster 0 osdmap e977: 8 total, 8 up, 8 in 2024-03-07T03:45:10.815 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:10 smithi145 bash[79878]: cluster 2024-03-07T03:45:09.572537+0000 mgr.y (mgr.34107) 999 : cluster 0 pgmap v1601: 105 pgs: 105 active+clean; 188 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 6.8 MiB/s wr, 33 op/s 2024-03-07T03:45:10.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:10 smithi145 bash[82329]: cluster 2024-03-07T03:45:09.537830+0000 mon.a (mon.0) 1784 : cluster 0 osdmap e977: 8 total, 8 up, 8 in 2024-03-07T03:45:10.815 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:10 smithi145 bash[82329]: cluster 2024-03-07T03:45:09.572537+0000 mgr.y (mgr.34107) 999 : cluster 0 pgmap v1601: 105 pgs: 105 active+clean; 188 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 6.8 MiB/s wr, 33 op/s 2024-03-07T03:45:10.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:10 smithi189 bash[66969]: cluster 2024-03-07T03:45:09.537830+0000 mon.a (mon.0) 1784 : cluster 0 osdmap e977: 8 total, 8 up, 8 in 2024-03-07T03:45:10.835 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:10 smithi189 bash[66969]: cluster 2024-03-07T03:45:09.572537+0000 mgr.y (mgr.34107) 999 : cluster 0 pgmap v1601: 105 pgs: 105 active+clean; 188 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 6.8 MiB/s wr, 33 op/s 2024-03-07T03:45:12.585 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:45:12 smithi189 bash[59814]: ts=2024-03-07T03:45:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:45:13.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:12 smithi145 bash[79878]: cluster 2024-03-07T03:45:11.573578+0000 mgr.y (mgr.34107) 1000 : cluster 0 pgmap v1602: 105 pgs: 105 active+clean; 188 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 6.8 MiB/s wr, 56 op/s; 1.3 MiB/s, 0 objects/s recovering 2024-03-07T03:45:13.065 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:45:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:45:12] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.43.0" 2024-03-07T03:45:13.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:12 smithi145 bash[82329]: cluster 2024-03-07T03:45:11.573578+0000 mgr.y (mgr.34107) 1000 : cluster 0 pgmap v1602: 105 pgs: 105 active+clean; 188 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 6.8 MiB/s wr, 56 op/s; 1.3 MiB/s, 0 objects/s recovering 2024-03-07T03:45:13.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:12 smithi189 bash[66969]: cluster 2024-03-07T03:45:11.573578+0000 mgr.y (mgr.34107) 1000 : cluster 0 pgmap v1602: 105 pgs: 105 active+clean; 188 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 6.8 MiB/s wr, 56 op/s; 1.3 MiB/s, 0 objects/s recovering 2024-03-07T03:45:15.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:14 smithi145 bash[82329]: cluster 2024-03-07T03:45:13.574322+0000 mgr.y (mgr.34107) 1001 : cluster 0 pgmap v1603: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 5.7 MiB/s wr, 45 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-03-07T03:45:15.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:14 smithi145 bash[79878]: cluster 2024-03-07T03:45:13.574322+0000 mgr.y (mgr.34107) 1001 : cluster 0 pgmap v1603: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 5.7 MiB/s wr, 45 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-03-07T03:45:15.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:14 smithi189 bash[66969]: cluster 2024-03-07T03:45:13.574322+0000 mgr.y (mgr.34107) 1001 : cluster 0 pgmap v1603: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 5.7 MiB/s wr, 45 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-03-07T03:45:17.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:16 smithi145 bash[79878]: cluster 2024-03-07T03:45:15.575169+0000 mgr.y (mgr.34107) 1002 : cluster 0 pgmap v1604: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 2.0 MiB/s wr, 46 op/s; 1024 KiB/s, 0 objects/s recovering 2024-03-07T03:45:17.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:16 smithi145 bash[82329]: cluster 2024-03-07T03:45:15.575169+0000 mgr.y (mgr.34107) 1002 : cluster 0 pgmap v1604: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 2.0 MiB/s wr, 46 op/s; 1024 KiB/s, 0 objects/s recovering 2024-03-07T03:45:17.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:16 smithi189 bash[66969]: cluster 2024-03-07T03:45:15.575169+0000 mgr.y (mgr.34107) 1002 : cluster 0 pgmap v1604: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 2.0 MiB/s wr, 46 op/s; 1024 KiB/s, 0 objects/s recovering 2024-03-07T03:45:19.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:18 smithi145 bash[79878]: cluster 2024-03-07T03:45:17.576376+0000 mgr.y (mgr.34107) 1003 : cluster 0 pgmap v1605: 105 pgs: 105 active+clean; 196 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 207 KiB/s rd, 1.8 MiB/s wr, 41 op/s; 905 KiB/s, 0 objects/s recovering 2024-03-07T03:45:19.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:18 smithi145 bash[79878]: audit 2024-03-07T03:45:17.676817+0000 mon.a (mon.0) 1785 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:19.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:18 smithi145 bash[82329]: cluster 2024-03-07T03:45:17.576376+0000 mgr.y (mgr.34107) 1003 : cluster 0 pgmap v1605: 105 pgs: 105 active+clean; 196 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 207 KiB/s rd, 1.8 MiB/s wr, 41 op/s; 905 KiB/s, 0 objects/s recovering 2024-03-07T03:45:19.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:18 smithi145 bash[82329]: audit 2024-03-07T03:45:17.676817+0000 mon.a (mon.0) 1785 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:19.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:18 smithi189 bash[66969]: cluster 2024-03-07T03:45:17.576376+0000 mgr.y (mgr.34107) 1003 : cluster 0 pgmap v1605: 105 pgs: 105 active+clean; 196 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 207 KiB/s rd, 1.8 MiB/s wr, 41 op/s; 905 KiB/s, 0 objects/s recovering 2024-03-07T03:45:19.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:18 smithi189 bash[66969]: audit 2024-03-07T03:45:17.676817+0000 mon.a (mon.0) 1785 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:20.120 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/7.Shrink (22286 ms) 2024-03-07T03:45:20.121 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/7.ShrinkToOne 2024-03-07T03:45:20.121 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:45:21.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:20 smithi145 bash[79878]: cluster 2024-03-07T03:45:19.577012+0000 mgr.y (mgr.34107) 1004 : cluster 0 pgmap v1606: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.0 MiB/s wr, 40 op/s; 816 KiB/s, 0 objects/s recovering 2024-03-07T03:45:21.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:20 smithi145 bash[82329]: cluster 2024-03-07T03:45:19.577012+0000 mgr.y (mgr.34107) 1004 : cluster 0 pgmap v1606: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.0 MiB/s wr, 40 op/s; 816 KiB/s, 0 objects/s recovering 2024-03-07T03:45:21.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:20 smithi189 bash[66969]: cluster 2024-03-07T03:45:19.577012+0000 mgr.y (mgr.34107) 1004 : cluster 0 pgmap v1606: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.0 MiB/s wr, 40 op/s; 816 KiB/s, 0 objects/s recovering 2024-03-07T03:45:22.585 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:45:22 smithi189 bash[59814]: ts=2024-03-07T03:45:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:45:23.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:22 smithi145 bash[79878]: cluster 2024-03-07T03:45:21.577958+0000 mgr.y (mgr.34107) 1005 : cluster 0 pgmap v1607: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.0 MiB/s wr, 39 op/s; 682 KiB/s, 0 objects/s recovering 2024-03-07T03:45:23.065 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:45:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:45:22] "GET /metrics HTTP/1.1" 200 36595 "" "Prometheus/2.43.0" 2024-03-07T03:45:23.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:22 smithi145 bash[82329]: cluster 2024-03-07T03:45:21.577958+0000 mgr.y (mgr.34107) 1005 : cluster 0 pgmap v1607: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.0 MiB/s wr, 39 op/s; 682 KiB/s, 0 objects/s recovering 2024-03-07T03:45:23.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:22 smithi189 bash[66969]: cluster 2024-03-07T03:45:21.577958+0000 mgr.y (mgr.34107) 1005 : cluster 0 pgmap v1607: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.0 MiB/s wr, 39 op/s; 682 KiB/s, 0 objects/s recovering 2024-03-07T03:45:25.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:24 smithi145 bash[79878]: cluster 2024-03-07T03:45:23.578755+0000 mgr.y (mgr.34107) 1006 : cluster 0 pgmap v1608: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 29 op/s 2024-03-07T03:45:25.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:24 smithi145 bash[82329]: cluster 2024-03-07T03:45:23.578755+0000 mgr.y (mgr.34107) 1006 : cluster 0 pgmap v1608: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 29 op/s 2024-03-07T03:45:25.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:24 smithi189 bash[66969]: cluster 2024-03-07T03:45:23.578755+0000 mgr.y (mgr.34107) 1006 : cluster 0 pgmap v1608: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 29 op/s 2024-03-07T03:45:27.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:26 smithi145 bash[79878]: cluster 2024-03-07T03:45:25.579343+0000 mgr.y (mgr.34107) 1007 : cluster 0 pgmap v1609: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 27 op/s 2024-03-07T03:45:27.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:26 smithi145 bash[82329]: cluster 2024-03-07T03:45:25.579343+0000 mgr.y (mgr.34107) 1007 : cluster 0 pgmap v1609: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 27 op/s 2024-03-07T03:45:27.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:26 smithi189 bash[66969]: cluster 2024-03-07T03:45:25.579343+0000 mgr.y (mgr.34107) 1007 : cluster 0 pgmap v1609: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 27 op/s 2024-03-07T03:45:29.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:28 smithi145 bash[79878]: cluster 2024-03-07T03:45:27.580033+0000 mgr.y (mgr.34107) 1008 : cluster 0 pgmap v1610: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 24 op/s 2024-03-07T03:45:29.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:28 smithi145 bash[82329]: cluster 2024-03-07T03:45:27.580033+0000 mgr.y (mgr.34107) 1008 : cluster 0 pgmap v1610: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 24 op/s 2024-03-07T03:45:29.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:28 smithi189 bash[66969]: cluster 2024-03-07T03:45:27.580033+0000 mgr.y (mgr.34107) 1008 : cluster 0 pgmap v1610: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 24 op/s 2024-03-07T03:45:30.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:29 smithi145 bash[79878]: cluster 2024-03-07T03:45:28.663532+0000 mon.a (mon.0) 1786 : cluster 0 osdmap e978: 8 total, 8 up, 8 in 2024-03-07T03:45:30.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:29 smithi145 bash[82329]: cluster 2024-03-07T03:45:28.663532+0000 mon.a (mon.0) 1786 : cluster 0 osdmap e978: 8 total, 8 up, 8 in 2024-03-07T03:45:30.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:29 smithi189 bash[66969]: cluster 2024-03-07T03:45:28.663532+0000 mon.a (mon.0) 1786 : cluster 0 osdmap e978: 8 total, 8 up, 8 in 2024-03-07T03:45:31.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:30 smithi145 bash[79878]: cluster 2024-03-07T03:45:29.580444+0000 mgr.y (mgr.34107) 1009 : cluster 0 pgmap v1612: 105 pgs: 105 active+clean; 240 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.6 MiB/s wr, 18 op/s 2024-03-07T03:45:31.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:30 smithi145 bash[82329]: cluster 2024-03-07T03:45:29.580444+0000 mgr.y (mgr.34107) 1009 : cluster 0 pgmap v1612: 105 pgs: 105 active+clean; 240 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.6 MiB/s wr, 18 op/s 2024-03-07T03:45:31.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:30 smithi189 bash[66969]: cluster 2024-03-07T03:45:29.580444+0000 mgr.y (mgr.34107) 1009 : cluster 0 pgmap v1612: 105 pgs: 105 active+clean; 240 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 3.6 MiB/s wr, 18 op/s 2024-03-07T03:45:32.585 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:45:32 smithi189 bash[59814]: ts=2024-03-07T03:45:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:45:33.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:32 smithi145 bash[79878]: cluster 2024-03-07T03:45:31.581002+0000 mgr.y (mgr.34107) 1010 : cluster 0 pgmap v1613: 105 pgs: 105 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 4.1 MiB/s wr, 30 op/s 2024-03-07T03:45:33.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:32 smithi145 bash[79878]: audit 2024-03-07T03:45:32.677009+0000 mon.a (mon.0) 1787 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:33.065 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:45:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:45:32] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2024-03-07T03:45:33.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:32 smithi145 bash[82329]: cluster 2024-03-07T03:45:31.581002+0000 mgr.y (mgr.34107) 1010 : cluster 0 pgmap v1613: 105 pgs: 105 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 4.1 MiB/s wr, 30 op/s 2024-03-07T03:45:33.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:32 smithi145 bash[82329]: audit 2024-03-07T03:45:32.677009+0000 mon.a (mon.0) 1787 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:33.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:32 smithi189 bash[66969]: cluster 2024-03-07T03:45:31.581002+0000 mgr.y (mgr.34107) 1010 : cluster 0 pgmap v1613: 105 pgs: 105 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 4.1 MiB/s wr, 30 op/s 2024-03-07T03:45:33.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:32 smithi189 bash[66969]: audit 2024-03-07T03:45:32.677009+0000 mon.a (mon.0) 1787 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:35.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:34 smithi145 bash[79878]: cluster 2024-03-07T03:45:33.581957+0000 mgr.y (mgr.34107) 1011 : cluster 0 pgmap v1614: 105 pgs: 105 active+clean; 261 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:35.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:34 smithi145 bash[82329]: cluster 2024-03-07T03:45:33.581957+0000 mgr.y (mgr.34107) 1011 : cluster 0 pgmap v1614: 105 pgs: 105 active+clean; 261 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:35.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:34 smithi189 bash[66969]: cluster 2024-03-07T03:45:33.581957+0000 mgr.y (mgr.34107) 1011 : cluster 0 pgmap v1614: 105 pgs: 105 active+clean; 261 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:37.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:36 smithi145 bash[79878]: cluster 2024-03-07T03:45:35.582583+0000 mgr.y (mgr.34107) 1012 : cluster 0 pgmap v1615: 105 pgs: 105 active+clean; 261 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:37.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:36 smithi145 bash[82329]: cluster 2024-03-07T03:45:35.582583+0000 mgr.y (mgr.34107) 1012 : cluster 0 pgmap v1615: 105 pgs: 105 active+clean; 261 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:37.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:36 smithi189 bash[66969]: cluster 2024-03-07T03:45:35.582583+0000 mgr.y (mgr.34107) 1012 : cluster 0 pgmap v1615: 105 pgs: 105 active+clean; 261 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:39.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:38 smithi145 bash[79878]: cluster 2024-03-07T03:45:37.583528+0000 mgr.y (mgr.34107) 1013 : cluster 0 pgmap v1616: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-03-07T03:45:39.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:38 smithi145 bash[82329]: cluster 2024-03-07T03:45:37.583528+0000 mgr.y (mgr.34107) 1013 : cluster 0 pgmap v1616: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-03-07T03:45:39.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:38 smithi189 bash[66969]: cluster 2024-03-07T03:45:37.583528+0000 mgr.y (mgr.34107) 1013 : cluster 0 pgmap v1616: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-03-07T03:45:41.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:40 smithi145 bash[82329]: cluster 2024-03-07T03:45:39.584285+0000 mgr.y (mgr.34107) 1014 : cluster 0 pgmap v1617: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 2.3 MiB/s wr, 30 op/s 2024-03-07T03:45:41.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:40 smithi145 bash[79878]: cluster 2024-03-07T03:45:39.584285+0000 mgr.y (mgr.34107) 1014 : cluster 0 pgmap v1617: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 2.3 MiB/s wr, 30 op/s 2024-03-07T03:45:41.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:40 smithi189 bash[66969]: cluster 2024-03-07T03:45:39.584285+0000 mgr.y (mgr.34107) 1014 : cluster 0 pgmap v1617: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 2.3 MiB/s wr, 30 op/s 2024-03-07T03:45:42.585 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:45:42 smithi189 bash[59814]: ts=2024-03-07T03:45:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:45:42.752 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/7.ShrinkToOne (22632 ms) 2024-03-07T03:45:42.753 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/7.ShrinkToOneAfterSnapshot 2024-03-07T03:45:42.753 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:45:43.065 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:45:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:45:42] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2024-03-07T03:45:43.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:42 smithi145 bash[82329]: cluster 2024-03-07T03:45:41.584847+0000 mgr.y (mgr.34107) 1015 : cluster 0 pgmap v1618: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 912 KiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-03-07T03:45:43.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:42 smithi145 bash[79878]: cluster 2024-03-07T03:45:41.584847+0000 mgr.y (mgr.34107) 1015 : cluster 0 pgmap v1618: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 912 KiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-03-07T03:45:43.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:42 smithi189 bash[66969]: cluster 2024-03-07T03:45:41.584847+0000 mgr.y (mgr.34107) 1015 : cluster 0 pgmap v1618: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 912 KiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-03-07T03:45:45.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:44 smithi145 bash[79878]: cluster 2024-03-07T03:45:43.585425+0000 mgr.y (mgr.34107) 1016 : cluster 0 pgmap v1619: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 906 KiB/s rd, 1.0 MiB/s wr, 25 op/s 2024-03-07T03:45:45.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:44 smithi145 bash[82329]: cluster 2024-03-07T03:45:43.585425+0000 mgr.y (mgr.34107) 1016 : cluster 0 pgmap v1619: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 906 KiB/s rd, 1.0 MiB/s wr, 25 op/s 2024-03-07T03:45:45.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:44 smithi189 bash[66969]: cluster 2024-03-07T03:45:43.585425+0000 mgr.y (mgr.34107) 1016 : cluster 0 pgmap v1619: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 906 KiB/s rd, 1.0 MiB/s wr, 25 op/s 2024-03-07T03:45:47.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:46 smithi145 bash[79878]: cluster 2024-03-07T03:45:45.586020+0000 mgr.y (mgr.34107) 1017 : cluster 0 pgmap v1620: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 901 KiB/s rd, 1.7 MiB/s wr, 19 op/s 2024-03-07T03:45:47.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:46 smithi145 bash[82329]: cluster 2024-03-07T03:45:45.586020+0000 mgr.y (mgr.34107) 1017 : cluster 0 pgmap v1620: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 901 KiB/s rd, 1.7 MiB/s wr, 19 op/s 2024-03-07T03:45:47.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:46 smithi189 bash[66969]: cluster 2024-03-07T03:45:45.586020+0000 mgr.y (mgr.34107) 1017 : cluster 0 pgmap v1620: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 901 KiB/s rd, 1.7 MiB/s wr, 19 op/s 2024-03-07T03:45:48.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:47 smithi145 bash[79878]: audit 2024-03-07T03:45:47.677615+0000 mon.a (mon.0) 1788 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:48.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:47 smithi145 bash[82329]: audit 2024-03-07T03:45:47.677615+0000 mon.a (mon.0) 1788 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:48.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:47 smithi189 bash[66969]: audit 2024-03-07T03:45:47.677615+0000 mon.a (mon.0) 1788 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:49.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:48 smithi145 bash[79878]: cluster 2024-03-07T03:45:47.586678+0000 mgr.y (mgr.34107) 1018 : cluster 0 pgmap v1621: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 904 KiB/s rd, 1.7 MiB/s wr, 23 op/s 2024-03-07T03:45:49.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:48 smithi145 bash[82329]: cluster 2024-03-07T03:45:47.586678+0000 mgr.y (mgr.34107) 1018 : cluster 0 pgmap v1621: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 904 KiB/s rd, 1.7 MiB/s wr, 23 op/s 2024-03-07T03:45:49.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:48 smithi189 bash[66969]: cluster 2024-03-07T03:45:47.586678+0000 mgr.y (mgr.34107) 1018 : cluster 0 pgmap v1621: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 904 KiB/s rd, 1.7 MiB/s wr, 23 op/s 2024-03-07T03:45:51.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:50 smithi145 bash[79878]: cluster 2024-03-07T03:45:49.587331+0000 mgr.y (mgr.34107) 1019 : cluster 0 pgmap v1622: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 854 KiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-03-07T03:45:51.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:50 smithi145 bash[82329]: cluster 2024-03-07T03:45:49.587331+0000 mgr.y (mgr.34107) 1019 : cluster 0 pgmap v1622: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 854 KiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-03-07T03:45:51.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:50 smithi189 bash[66969]: cluster 2024-03-07T03:45:49.587331+0000 mgr.y (mgr.34107) 1019 : cluster 0 pgmap v1622: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 854 KiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-03-07T03:45:52.585 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:45:52 smithi189 bash[59814]: ts=2024-03-07T03:45:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:45:53.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:52 smithi145 bash[79878]: cluster 2024-03-07T03:45:51.588042+0000 mgr.y (mgr.34107) 1020 : cluster 0 pgmap v1623: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 852 KiB/s rd, 1.3 MiB/s wr, 18 op/s 2024-03-07T03:45:53.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:45:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:45:52] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.43.0" 2024-03-07T03:45:53.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:52 smithi145 bash[82329]: cluster 2024-03-07T03:45:51.588042+0000 mgr.y (mgr.34107) 1020 : cluster 0 pgmap v1623: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 852 KiB/s rd, 1.3 MiB/s wr, 18 op/s 2024-03-07T03:45:53.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:52 smithi189 bash[66969]: cluster 2024-03-07T03:45:51.588042+0000 mgr.y (mgr.34107) 1020 : cluster 0 pgmap v1623: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 852 KiB/s rd, 1.3 MiB/s wr, 18 op/s 2024-03-07T03:45:54.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:53 smithi145 bash[79878]: cluster 2024-03-07T03:45:52.746368+0000 mon.a (mon.0) 1789 : cluster 0 osdmap e979: 8 total, 8 up, 8 in 2024-03-07T03:45:54.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:53 smithi145 bash[82329]: cluster 2024-03-07T03:45:52.746368+0000 mon.a (mon.0) 1789 : cluster 0 osdmap e979: 8 total, 8 up, 8 in 2024-03-07T03:45:54.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:53 smithi189 bash[66969]: cluster 2024-03-07T03:45:52.746368+0000 mon.a (mon.0) 1789 : cluster 0 osdmap e979: 8 total, 8 up, 8 in 2024-03-07T03:45:55.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:54 smithi145 bash[79878]: cluster 2024-03-07T03:45:53.588905+0000 mgr.y (mgr.34107) 1021 : cluster 0 pgmap v1625: 105 pgs: 105 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 2.1 MiB/s wr, 12 op/s 2024-03-07T03:45:55.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:54 smithi145 bash[82329]: cluster 2024-03-07T03:45:53.588905+0000 mgr.y (mgr.34107) 1021 : cluster 0 pgmap v1625: 105 pgs: 105 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 2.1 MiB/s wr, 12 op/s 2024-03-07T03:45:55.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:54 smithi189 bash[66969]: cluster 2024-03-07T03:45:53.588905+0000 mgr.y (mgr.34107) 1021 : cluster 0 pgmap v1625: 105 pgs: 105 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 2.1 MiB/s wr, 12 op/s 2024-03-07T03:45:57.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:56 smithi145 bash[79878]: cluster 2024-03-07T03:45:55.589701+0000 mgr.y (mgr.34107) 1022 : cluster 0 pgmap v1626: 105 pgs: 105 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 1.7 MiB/s wr, 11 op/s 2024-03-07T03:45:57.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:56 smithi145 bash[82329]: cluster 2024-03-07T03:45:55.589701+0000 mgr.y (mgr.34107) 1022 : cluster 0 pgmap v1626: 105 pgs: 105 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 1.7 MiB/s wr, 11 op/s 2024-03-07T03:45:57.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:56 smithi189 bash[66969]: cluster 2024-03-07T03:45:55.589701+0000 mgr.y (mgr.34107) 1022 : cluster 0 pgmap v1626: 105 pgs: 105 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 1.7 MiB/s wr, 11 op/s 2024-03-07T03:45:59.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:45:58 smithi145 bash[79878]: cluster 2024-03-07T03:45:57.590790+0000 mgr.y (mgr.34107) 1023 : cluster 0 pgmap v1627: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:59.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:45:58 smithi145 bash[82329]: cluster 2024-03-07T03:45:57.590790+0000 mgr.y (mgr.34107) 1023 : cluster 0 pgmap v1627: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:59.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:45:58 smithi189 bash[66969]: cluster 2024-03-07T03:45:57.590790+0000 mgr.y (mgr.34107) 1023 : cluster 0 pgmap v1627: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:01.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:00 smithi145 bash[79878]: cluster 2024-03-07T03:45:59.591460+0000 mgr.y (mgr.34107) 1024 : cluster 0 pgmap v1628: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:01.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:00 smithi145 bash[82329]: cluster 2024-03-07T03:45:59.591460+0000 mgr.y (mgr.34107) 1024 : cluster 0 pgmap v1628: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:01.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:00 smithi189 bash[66969]: cluster 2024-03-07T03:45:59.591460+0000 mgr.y (mgr.34107) 1024 : cluster 0 pgmap v1628: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:02.586 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:46:02 smithi189 bash[59814]: ts=2024-03-07T03:46:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:46:03.065 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:46:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:46:02] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2024-03-07T03:46:03.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:02 smithi145 bash[79878]: cluster 2024-03-07T03:46:01.592428+0000 mgr.y (mgr.34107) 1025 : cluster 0 pgmap v1629: 105 pgs: 105 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 4.5 MiB/s wr, 42 op/s 2024-03-07T03:46:03.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:02 smithi145 bash[79878]: audit 2024-03-07T03:46:02.677652+0000 mon.a (mon.0) 1790 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:03.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:02 smithi145 bash[82329]: cluster 2024-03-07T03:46:01.592428+0000 mgr.y (mgr.34107) 1025 : cluster 0 pgmap v1629: 105 pgs: 105 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 4.5 MiB/s wr, 42 op/s 2024-03-07T03:46:03.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:02 smithi145 bash[82329]: audit 2024-03-07T03:46:02.677652+0000 mon.a (mon.0) 1790 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:03.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:02 smithi189 bash[66969]: cluster 2024-03-07T03:46:01.592428+0000 mgr.y (mgr.34107) 1025 : cluster 0 pgmap v1629: 105 pgs: 105 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 117 KiB/s rd, 4.5 MiB/s wr, 42 op/s 2024-03-07T03:46:03.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:02 smithi189 bash[66969]: audit 2024-03-07T03:46:02.677652+0000 mon.a (mon.0) 1790 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:05.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:04 smithi145 bash[82329]: cluster 2024-03-07T03:46:03.593187+0000 mgr.y (mgr.34107) 1026 : cluster 0 pgmap v1630: 105 pgs: 105 active+clean; 318 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 204 KiB/s rd, 4.5 MiB/s wr, 42 op/s 2024-03-07T03:46:05.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:04 smithi145 bash[79878]: cluster 2024-03-07T03:46:03.593187+0000 mgr.y (mgr.34107) 1026 : cluster 0 pgmap v1630: 105 pgs: 105 active+clean; 318 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 204 KiB/s rd, 4.5 MiB/s wr, 42 op/s 2024-03-07T03:46:05.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:04 smithi189 bash[66969]: cluster 2024-03-07T03:46:03.593187+0000 mgr.y (mgr.34107) 1026 : cluster 0 pgmap v1630: 105 pgs: 105 active+clean; 318 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 204 KiB/s rd, 4.5 MiB/s wr, 42 op/s 2024-03-07T03:46:07.065 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:06 smithi145 bash[82329]: cluster 2024-03-07T03:46:05.594001+0000 mgr.y (mgr.34107) 1027 : cluster 0 pgmap v1631: 105 pgs: 105 active+clean; 318 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 3.0 MiB/s wr, 32 op/s 2024-03-07T03:46:07.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:06 smithi145 bash[82329]: cluster 2024-03-07T03:46:05.790256+0000 mon.a (mon.0) 1791 : cluster 0 osdmap e980: 8 total, 8 up, 8 in 2024-03-07T03:46:07.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:06 smithi145 bash[79878]: cluster 2024-03-07T03:46:05.594001+0000 mgr.y (mgr.34107) 1027 : cluster 0 pgmap v1631: 105 pgs: 105 active+clean; 318 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 3.0 MiB/s wr, 32 op/s 2024-03-07T03:46:07.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:06 smithi145 bash[79878]: cluster 2024-03-07T03:46:05.790256+0000 mon.a (mon.0) 1791 : cluster 0 osdmap e980: 8 total, 8 up, 8 in 2024-03-07T03:46:07.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:06 smithi189 bash[66969]: cluster 2024-03-07T03:46:05.594001+0000 mgr.y (mgr.34107) 1027 : cluster 0 pgmap v1631: 105 pgs: 105 active+clean; 318 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 3.0 MiB/s wr, 32 op/s 2024-03-07T03:46:07.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:06 smithi189 bash[66969]: cluster 2024-03-07T03:46:05.790256+0000 mon.a (mon.0) 1791 : cluster 0 osdmap e980: 8 total, 8 up, 8 in 2024-03-07T03:46:08.259 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/7.ShrinkToOneAfterSnapshot (25506 ms) 2024-03-07T03:46:08.260 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/7.MinOverlap 2024-03-07T03:46:08.260 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:46:09.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:08 smithi145 bash[79878]: cluster 2024-03-07T03:46:07.594868+0000 mgr.y (mgr.34107) 1028 : cluster 0 pgmap v1633: 105 pgs: 105 active+clean; 346 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.6 MiB/s wr, 25 op/s 2024-03-07T03:46:09.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:08 smithi145 bash[79878]: audit 2024-03-07T03:46:08.212251+0000 mon.a (mon.0) 1792 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:46:09.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:08 smithi145 bash[82329]: cluster 2024-03-07T03:46:07.594868+0000 mgr.y (mgr.34107) 1028 : cluster 0 pgmap v1633: 105 pgs: 105 active+clean; 346 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.6 MiB/s wr, 25 op/s 2024-03-07T03:46:09.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:08 smithi145 bash[82329]: audit 2024-03-07T03:46:08.212251+0000 mon.a (mon.0) 1792 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:46:09.085 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:08 smithi189 bash[66969]: cluster 2024-03-07T03:46:07.594868+0000 mgr.y (mgr.34107) 1028 : cluster 0 pgmap v1633: 105 pgs: 105 active+clean; 346 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.6 MiB/s wr, 25 op/s 2024-03-07T03:46:09.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:08 smithi189 bash[66969]: audit 2024-03-07T03:46:08.212251+0000 mon.a (mon.0) 1792 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:46:11.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:10 smithi145 bash[79878]: cluster 2024-03-07T03:46:09.595620+0000 mgr.y (mgr.34107) 1029 : cluster 0 pgmap v1634: 105 pgs: 105 active+clean; 358 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 4.8 MiB/s wr, 40 op/s 2024-03-07T03:46:11.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:10 smithi145 bash[82329]: cluster 2024-03-07T03:46:09.595620+0000 mgr.y (mgr.34107) 1029 : cluster 0 pgmap v1634: 105 pgs: 105 active+clean; 358 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 4.8 MiB/s wr, 40 op/s 2024-03-07T03:46:11.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:10 smithi189 bash[66969]: cluster 2024-03-07T03:46:09.595620+0000 mgr.y (mgr.34107) 1029 : cluster 0 pgmap v1634: 105 pgs: 105 active+clean; 358 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 4.8 MiB/s wr, 40 op/s 2024-03-07T03:46:12.586 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:46:12 smithi189 bash[59814]: ts=2024-03-07T03:46:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:46:13.065 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:12 smithi145 bash[79878]: cluster 2024-03-07T03:46:11.596313+0000 mgr.y (mgr.34107) 1030 : cluster 0 pgmap v1635: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 39 op/s 2024-03-07T03:46:13.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:12 smithi145 bash[82329]: cluster 2024-03-07T03:46:11.596313+0000 mgr.y (mgr.34107) 1030 : cluster 0 pgmap v1635: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 39 op/s 2024-03-07T03:46:13.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:46:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:46:12] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2024-03-07T03:46:13.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:12 smithi189 bash[66969]: cluster 2024-03-07T03:46:11.596313+0000 mgr.y (mgr.34107) 1030 : cluster 0 pgmap v1635: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 39 op/s 2024-03-07T03:46:15.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:14 smithi145 bash[79878]: cluster 2024-03-07T03:46:13.597008+0000 mgr.y (mgr.34107) 1031 : cluster 0 pgmap v1636: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 5.6 MiB/s wr, 35 op/s 2024-03-07T03:46:15.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:14 smithi145 bash[79878]: audit 2024-03-07T03:46:14.201609+0000 mon.a (mon.0) 1793 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:15.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:14 smithi145 bash[79878]: audit 2024-03-07T03:46:14.209271+0000 mon.a (mon.0) 1794 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:15.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:14 smithi145 bash[79878]: audit 2024-03-07T03:46:14.304536+0000 mon.a (mon.0) 1795 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:15.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:14 smithi145 bash[79878]: audit 2024-03-07T03:46:14.310536+0000 mon.a (mon.0) 1796 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:15.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:14 smithi145 bash[82329]: cluster 2024-03-07T03:46:13.597008+0000 mgr.y (mgr.34107) 1031 : cluster 0 pgmap v1636: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 5.6 MiB/s wr, 35 op/s 2024-03-07T03:46:15.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:14 smithi145 bash[82329]: audit 2024-03-07T03:46:14.201609+0000 mon.a (mon.0) 1793 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:15.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:14 smithi145 bash[82329]: audit 2024-03-07T03:46:14.209271+0000 mon.a (mon.0) 1794 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:15.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:14 smithi145 bash[82329]: audit 2024-03-07T03:46:14.304536+0000 mon.a (mon.0) 1795 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:15.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:14 smithi145 bash[82329]: audit 2024-03-07T03:46:14.310536+0000 mon.a (mon.0) 1796 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:15.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:14 smithi189 bash[66969]: cluster 2024-03-07T03:46:13.597008+0000 mgr.y (mgr.34107) 1031 : cluster 0 pgmap v1636: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 5.6 MiB/s wr, 35 op/s 2024-03-07T03:46:15.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:14 smithi189 bash[66969]: audit 2024-03-07T03:46:14.201609+0000 mon.a (mon.0) 1793 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:15.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:14 smithi189 bash[66969]: audit 2024-03-07T03:46:14.209271+0000 mon.a (mon.0) 1794 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:15.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:14 smithi189 bash[66969]: audit 2024-03-07T03:46:14.304536+0000 mon.a (mon.0) 1795 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:15.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:14 smithi189 bash[66969]: audit 2024-03-07T03:46:14.310536+0000 mon.a (mon.0) 1796 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:16.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:15 smithi145 bash[79878]: audit 2024-03-07T03:46:14.940692+0000 mon.a (mon.0) 1797 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:46:16.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:15 smithi145 bash[79878]: audit 2024-03-07T03:46:14.941912+0000 mon.a (mon.0) 1798 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:46:16.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:15 smithi145 bash[79878]: audit 2024-03-07T03:46:14.951409+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:16.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:15 smithi145 bash[82329]: audit 2024-03-07T03:46:14.940692+0000 mon.a (mon.0) 1797 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:46:16.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:15 smithi145 bash[82329]: audit 2024-03-07T03:46:14.941912+0000 mon.a (mon.0) 1798 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:46:16.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:15 smithi145 bash[82329]: audit 2024-03-07T03:46:14.951409+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:16.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:15 smithi189 bash[66969]: audit 2024-03-07T03:46:14.940692+0000 mon.a (mon.0) 1797 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:46:16.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:15 smithi189 bash[66969]: audit 2024-03-07T03:46:14.941912+0000 mon.a (mon.0) 1798 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:46:16.086 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:15 smithi189 bash[66969]: audit 2024-03-07T03:46:14.951409+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:46:17.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:16 smithi145 bash[79878]: cluster 2024-03-07T03:46:15.597836+0000 mgr.y (mgr.34107) 1032 : cluster 0 pgmap v1637: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 5.6 MiB/s wr, 35 op/s 2024-03-07T03:46:17.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:16 smithi145 bash[79878]: cluster 2024-03-07T03:46:15.964492+0000 mon.a (mon.0) 1800 : cluster 0 osdmap e981: 8 total, 8 up, 8 in 2024-03-07T03:46:17.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:16 smithi145 bash[82329]: cluster 2024-03-07T03:46:15.597836+0000 mgr.y (mgr.34107) 1032 : cluster 0 pgmap v1637: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 5.6 MiB/s wr, 35 op/s 2024-03-07T03:46:17.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:16 smithi145 bash[82329]: cluster 2024-03-07T03:46:15.964492+0000 mon.a (mon.0) 1800 : cluster 0 osdmap e981: 8 total, 8 up, 8 in 2024-03-07T03:46:17.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:16 smithi189 bash[66969]: cluster 2024-03-07T03:46:15.597836+0000 mgr.y (mgr.34107) 1032 : cluster 0 pgmap v1637: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 5.6 MiB/s wr, 35 op/s 2024-03-07T03:46:17.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:16 smithi189 bash[66969]: cluster 2024-03-07T03:46:15.964492+0000 mon.a (mon.0) 1800 : cluster 0 osdmap e981: 8 total, 8 up, 8 in 2024-03-07T03:46:18.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:17 smithi145 bash[79878]: audit 2024-03-07T03:46:17.677716+0000 mon.a (mon.0) 1801 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:18.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:17 smithi145 bash[82329]: audit 2024-03-07T03:46:17.677716+0000 mon.a (mon.0) 1801 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:18.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:17 smithi189 bash[66969]: audit 2024-03-07T03:46:17.677716+0000 mon.a (mon.0) 1801 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:19.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:18 smithi145 bash[79878]: cluster 2024-03-07T03:46:17.598725+0000 mgr.y (mgr.34107) 1033 : cluster 0 pgmap v1639: 105 pgs: 105 active+clean; 398 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.2 MiB/s wr, 44 op/s 2024-03-07T03:46:19.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:18 smithi145 bash[82329]: cluster 2024-03-07T03:46:17.598725+0000 mgr.y (mgr.34107) 1033 : cluster 0 pgmap v1639: 105 pgs: 105 active+clean; 398 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.2 MiB/s wr, 44 op/s 2024-03-07T03:46:19.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:18 smithi189 bash[66969]: cluster 2024-03-07T03:46:17.598725+0000 mgr.y (mgr.34107) 1033 : cluster 0 pgmap v1639: 105 pgs: 105 active+clean; 398 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.2 MiB/s wr, 44 op/s 2024-03-07T03:46:21.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:20 smithi145 bash[79878]: cluster 2024-03-07T03:46:19.599599+0000 mgr.y (mgr.34107) 1034 : cluster 0 pgmap v1640: 105 pgs: 105 active+clean; 407 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.9 MiB/s wr, 39 op/s 2024-03-07T03:46:21.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:20 smithi145 bash[82329]: cluster 2024-03-07T03:46:19.599599+0000 mgr.y (mgr.34107) 1034 : cluster 0 pgmap v1640: 105 pgs: 105 active+clean; 407 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.9 MiB/s wr, 39 op/s 2024-03-07T03:46:21.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:20 smithi189 bash[66969]: cluster 2024-03-07T03:46:19.599599+0000 mgr.y (mgr.34107) 1034 : cluster 0 pgmap v1640: 105 pgs: 105 active+clean; 407 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.9 MiB/s wr, 39 op/s 2024-03-07T03:46:22.586 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:46:22 smithi189 bash[59814]: ts=2024-03-07T03:46:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:46:23.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:22 smithi145 bash[79878]: cluster 2024-03-07T03:46:21.600554+0000 mgr.y (mgr.34107) 1035 : cluster 0 pgmap v1641: 105 pgs: 105 active+clean; 415 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:46:23.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:22 smithi145 bash[82329]: cluster 2024-03-07T03:46:21.600554+0000 mgr.y (mgr.34107) 1035 : cluster 0 pgmap v1641: 105 pgs: 105 active+clean; 415 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:46:23.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:46:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:46:22] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.43.0" 2024-03-07T03:46:23.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:22 smithi189 bash[66969]: cluster 2024-03-07T03:46:21.600554+0000 mgr.y (mgr.34107) 1035 : cluster 0 pgmap v1641: 105 pgs: 105 active+clean; 415 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:46:25.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:24 smithi145 bash[82329]: cluster 2024-03-07T03:46:23.601234+0000 mgr.y (mgr.34107) 1036 : cluster 0 pgmap v1642: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:46:25.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:24 smithi145 bash[79878]: cluster 2024-03-07T03:46:23.601234+0000 mgr.y (mgr.34107) 1036 : cluster 0 pgmap v1642: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:46:25.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:24 smithi189 bash[66969]: cluster 2024-03-07T03:46:23.601234+0000 mgr.y (mgr.34107) 1036 : cluster 0 pgmap v1642: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:46:27.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:26 smithi145 bash[79878]: cluster 2024-03-07T03:46:25.601982+0000 mgr.y (mgr.34107) 1037 : cluster 0 pgmap v1643: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:46:27.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:26 smithi145 bash[82329]: cluster 2024-03-07T03:46:25.601982+0000 mgr.y (mgr.34107) 1037 : cluster 0 pgmap v1643: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:46:27.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:26 smithi189 bash[66969]: cluster 2024-03-07T03:46:25.601982+0000 mgr.y (mgr.34107) 1037 : cluster 0 pgmap v1643: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:46:29.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:28 smithi145 bash[82329]: cluster 2024-03-07T03:46:27.603020+0000 mgr.y (mgr.34107) 1038 : cluster 0 pgmap v1644: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 3.9 MiB/s wr, 39 op/s 2024-03-07T03:46:29.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:28 smithi145 bash[79878]: cluster 2024-03-07T03:46:27.603020+0000 mgr.y (mgr.34107) 1038 : cluster 0 pgmap v1644: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 3.9 MiB/s wr, 39 op/s 2024-03-07T03:46:29.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:28 smithi189 bash[66969]: cluster 2024-03-07T03:46:27.603020+0000 mgr.y (mgr.34107) 1038 : cluster 0 pgmap v1644: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 3.9 MiB/s wr, 39 op/s 2024-03-07T03:46:30.041 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/7.MinOverlap (21781 ms) 2024-03-07T03:46:30.041 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] EncryptedFlattenTest/7.ZeroOverlap 2024-03-07T03:46:30.041 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:46:31.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:30 smithi145 bash[79878]: cluster 2024-03-07T03:46:29.603588+0000 mgr.y (mgr.34107) 1039 : cluster 0 pgmap v1645: 105 pgs: 105 active+clean; 427 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.1 MiB/s wr, 20 op/s 2024-03-07T03:46:31.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:30 smithi145 bash[82329]: cluster 2024-03-07T03:46:29.603588+0000 mgr.y (mgr.34107) 1039 : cluster 0 pgmap v1645: 105 pgs: 105 active+clean; 427 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.1 MiB/s wr, 20 op/s 2024-03-07T03:46:31.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:30 smithi189 bash[66969]: cluster 2024-03-07T03:46:29.603588+0000 mgr.y (mgr.34107) 1039 : cluster 0 pgmap v1645: 105 pgs: 105 active+clean; 427 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.1 MiB/s wr, 20 op/s 2024-03-07T03:46:32.586 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:46:32 smithi189 bash[59814]: ts=2024-03-07T03:46:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:46:33.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:32 smithi145 bash[79878]: cluster 2024-03-07T03:46:31.604375+0000 mgr.y (mgr.34107) 1040 : cluster 0 pgmap v1646: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-03-07T03:46:33.066 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:32 smithi145 bash[79878]: audit 2024-03-07T03:46:32.678342+0000 mon.a (mon.0) 1802 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:33.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:32 smithi145 bash[82329]: cluster 2024-03-07T03:46:31.604375+0000 mgr.y (mgr.34107) 1040 : cluster 0 pgmap v1646: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-03-07T03:46:33.066 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:32 smithi145 bash[82329]: audit 2024-03-07T03:46:32.678342+0000 mon.a (mon.0) 1802 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:33.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:46:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:46:32] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2024-03-07T03:46:33.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:32 smithi189 bash[66969]: cluster 2024-03-07T03:46:31.604375+0000 mgr.y (mgr.34107) 1040 : cluster 0 pgmap v1646: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-03-07T03:46:33.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:32 smithi189 bash[66969]: audit 2024-03-07T03:46:32.678342+0000 mon.a (mon.0) 1802 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:35.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:35 smithi145 bash[79878]: cluster 2024-03-07T03:46:33.605105+0000 mgr.y (mgr.34107) 1041 : cluster 0 pgmap v1647: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 22 op/s 2024-03-07T03:46:35.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:35 smithi145 bash[82329]: cluster 2024-03-07T03:46:33.605105+0000 mgr.y (mgr.34107) 1041 : cluster 0 pgmap v1647: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 22 op/s 2024-03-07T03:46:35.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:35 smithi189 bash[66969]: cluster 2024-03-07T03:46:33.605105+0000 mgr.y (mgr.34107) 1041 : cluster 0 pgmap v1647: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 22 op/s 2024-03-07T03:46:37.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:37 smithi145 bash[79878]: cluster 2024-03-07T03:46:35.605683+0000 mgr.y (mgr.34107) 1042 : cluster 0 pgmap v1648: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 24 op/s 2024-03-07T03:46:37.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:37 smithi145 bash[82329]: cluster 2024-03-07T03:46:35.605683+0000 mgr.y (mgr.34107) 1042 : cluster 0 pgmap v1648: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 24 op/s 2024-03-07T03:46:37.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:37 smithi189 bash[66969]: cluster 2024-03-07T03:46:35.605683+0000 mgr.y (mgr.34107) 1042 : cluster 0 pgmap v1648: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 24 op/s 2024-03-07T03:46:39.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:39 smithi145 bash[79878]: cluster 2024-03-07T03:46:37.606395+0000 mgr.y (mgr.34107) 1043 : cluster 0 pgmap v1649: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 24 op/s 2024-03-07T03:46:39.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:39 smithi145 bash[82329]: cluster 2024-03-07T03:46:37.606395+0000 mgr.y (mgr.34107) 1043 : cluster 0 pgmap v1649: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 24 op/s 2024-03-07T03:46:39.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:39 smithi189 bash[66969]: cluster 2024-03-07T03:46:37.606395+0000 mgr.y (mgr.34107) 1043 : cluster 0 pgmap v1649: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 24 op/s 2024-03-07T03:46:41.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:41 smithi145 bash[79878]: cluster 2024-03-07T03:46:39.607280+0000 mgr.y (mgr.34107) 1044 : cluster 0 pgmap v1650: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.0 MiB/s wr, 17 op/s 2024-03-07T03:46:41.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:41 smithi145 bash[79878]: cluster 2024-03-07T03:46:40.024364+0000 mon.a (mon.0) 1803 : cluster 0 osdmap e982: 8 total, 8 up, 8 in 2024-03-07T03:46:41.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:41 smithi145 bash[82329]: cluster 2024-03-07T03:46:39.607280+0000 mgr.y (mgr.34107) 1044 : cluster 0 pgmap v1650: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.0 MiB/s wr, 17 op/s 2024-03-07T03:46:41.317 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:41 smithi145 bash[82329]: cluster 2024-03-07T03:46:40.024364+0000 mon.a (mon.0) 1803 : cluster 0 osdmap e982: 8 total, 8 up, 8 in 2024-03-07T03:46:41.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:41 smithi189 bash[66969]: cluster 2024-03-07T03:46:39.607280+0000 mgr.y (mgr.34107) 1044 : cluster 0 pgmap v1650: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.0 MiB/s wr, 17 op/s 2024-03-07T03:46:41.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:41 smithi189 bash[66969]: cluster 2024-03-07T03:46:40.024364+0000 mon.a (mon.0) 1803 : cluster 0 osdmap e982: 8 total, 8 up, 8 in 2024-03-07T03:46:42.586 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:46:42 smithi189 bash[59814]: ts=2024-03-07T03:46:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:46:43.017 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:46:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:46:42] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2024-03-07T03:46:43.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:43 smithi145 bash[79878]: cluster 2024-03-07T03:46:41.608166+0000 mgr.y (mgr.34107) 1045 : cluster 0 pgmap v1652: 105 pgs: 105 active+clean; 448 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 432 KiB/s rd, 2.5 MiB/s wr, 30 op/s 2024-03-07T03:46:43.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:43 smithi145 bash[82329]: cluster 2024-03-07T03:46:41.608166+0000 mgr.y (mgr.34107) 1045 : cluster 0 pgmap v1652: 105 pgs: 105 active+clean; 448 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 432 KiB/s rd, 2.5 MiB/s wr, 30 op/s 2024-03-07T03:46:43.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:43 smithi189 bash[66969]: cluster 2024-03-07T03:46:41.608166+0000 mgr.y (mgr.34107) 1045 : cluster 0 pgmap v1652: 105 pgs: 105 active+clean; 448 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 432 KiB/s rd, 2.5 MiB/s wr, 30 op/s 2024-03-07T03:46:45.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:45 smithi145 bash[79878]: cluster 2024-03-07T03:46:43.609144+0000 mgr.y (mgr.34107) 1046 : cluster 0 pgmap v1653: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 4.1 MiB/s wr, 24 op/s 2024-03-07T03:46:45.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:45 smithi145 bash[82329]: cluster 2024-03-07T03:46:43.609144+0000 mgr.y (mgr.34107) 1046 : cluster 0 pgmap v1653: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 4.1 MiB/s wr, 24 op/s 2024-03-07T03:46:45.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:45 smithi189 bash[66969]: cluster 2024-03-07T03:46:43.609144+0000 mgr.y (mgr.34107) 1046 : cluster 0 pgmap v1653: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 4.1 MiB/s wr, 24 op/s 2024-03-07T03:46:47.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:47 smithi145 bash[79878]: cluster 2024-03-07T03:46:45.609862+0000 mgr.y (mgr.34107) 1047 : cluster 0 pgmap v1654: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:47.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:47 smithi145 bash[82329]: cluster 2024-03-07T03:46:45.609862+0000 mgr.y (mgr.34107) 1047 : cluster 0 pgmap v1654: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:47.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:47 smithi189 bash[66969]: cluster 2024-03-07T03:46:45.609862+0000 mgr.y (mgr.34107) 1047 : cluster 0 pgmap v1654: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:48.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:48 smithi145 bash[79878]: audit 2024-03-07T03:46:47.678556+0000 mon.a (mon.0) 1804 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:48.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:48 smithi145 bash[82329]: audit 2024-03-07T03:46:47.678556+0000 mon.a (mon.0) 1804 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:48.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:48 smithi189 bash[66969]: audit 2024-03-07T03:46:47.678556+0000 mon.a (mon.0) 1804 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:49.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:49 smithi145 bash[79878]: cluster 2024-03-07T03:46:47.610625+0000 mgr.y (mgr.34107) 1048 : cluster 0 pgmap v1655: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:46:49.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:49 smithi145 bash[82329]: cluster 2024-03-07T03:46:47.610625+0000 mgr.y (mgr.34107) 1048 : cluster 0 pgmap v1655: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:46:49.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:49 smithi189 bash[66969]: cluster 2024-03-07T03:46:47.610625+0000 mgr.y (mgr.34107) 1048 : cluster 0 pgmap v1655: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:46:51.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:51 smithi145 bash[79878]: cluster 2024-03-07T03:46:49.611490+0000 mgr.y (mgr.34107) 1049 : cluster 0 pgmap v1656: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 4.9 MiB/s wr, 37 op/s 2024-03-07T03:46:51.317 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:51 smithi145 bash[82329]: cluster 2024-03-07T03:46:49.611490+0000 mgr.y (mgr.34107) 1049 : cluster 0 pgmap v1656: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 4.9 MiB/s wr, 37 op/s 2024-03-07T03:46:51.337 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:51 smithi189 bash[66969]: cluster 2024-03-07T03:46:49.611490+0000 mgr.y (mgr.34107) 1049 : cluster 0 pgmap v1656: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 4.9 MiB/s wr, 37 op/s 2024-03-07T03:46:52.587 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:46:52 smithi189 bash[59814]: ts=2024-03-07T03:46:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:46:53.052 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:46:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:46:52] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2024-03-07T03:46:53.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:53 smithi145 bash[82329]: cluster 2024-03-07T03:46:51.612230+0000 mgr.y (mgr.34107) 1050 : cluster 0 pgmap v1657: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 189 KiB/s rd, 4.2 MiB/s wr, 37 op/s 2024-03-07T03:46:53.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:53 smithi145 bash[79878]: cluster 2024-03-07T03:46:51.612230+0000 mgr.y (mgr.34107) 1050 : cluster 0 pgmap v1657: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 189 KiB/s rd, 4.2 MiB/s wr, 37 op/s 2024-03-07T03:46:53.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:53 smithi189 bash[66969]: cluster 2024-03-07T03:46:51.612230+0000 mgr.y (mgr.34107) 1050 : cluster 0 pgmap v1657: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 189 KiB/s rd, 4.2 MiB/s wr, 37 op/s 2024-03-07T03:46:54.642 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] EncryptedFlattenTest/7.ZeroOverlap (24601 ms) 2024-03-07T03:46:55.084 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 7 tests from EncryptedFlattenTest/7 (167912 ms total) 2024-03-07T03:46:55.084 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:46:55.084 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 13 tests from DiffIterateTest/0, where TypeParam = DiffIterateParams 2024-03-07T03:46:55.135 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/0.DiffIterate 2024-03-07T03:46:55.316 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:55 smithi145 bash[79878]: cluster 2024-03-07T03:46:53.613192+0000 mgr.y (mgr.34107) 1051 : cluster 0 pgmap v1658: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 564 KiB/s rd, 2.3 MiB/s wr, 23 op/s 2024-03-07T03:46:55.316 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:55 smithi145 bash[82329]: cluster 2024-03-07T03:46:53.613192+0000 mgr.y (mgr.34107) 1051 : cluster 0 pgmap v1658: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 564 KiB/s rd, 2.3 MiB/s wr, 23 op/s 2024-03-07T03:46:55.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:55 smithi189 bash[66969]: cluster 2024-03-07T03:46:53.613192+0000 mgr.y (mgr.34107) 1051 : cluster 0 pgmap v1658: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 564 KiB/s rd, 2.3 MiB/s wr, 23 op/s 2024-03-07T03:46:56.336 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:56 smithi189 bash[66969]: cluster 2024-03-07T03:46:55.074788+0000 mon.a (mon.0) 1805 : cluster 0 osdmap e983: 8 total, 8 up, 8 in 2024-03-07T03:46:56.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:56 smithi145 bash[79878]: cluster 2024-03-07T03:46:55.074788+0000 mon.a (mon.0) 1805 : cluster 0 osdmap e983: 8 total, 8 up, 8 in 2024-03-07T03:46:56.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:56 smithi145 bash[82329]: cluster 2024-03-07T03:46:55.074788+0000 mon.a (mon.0) 1805 : cluster 0 osdmap e983: 8 total, 8 up, 8 in 2024-03-07T03:46:57.104 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:46:57.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:57 smithi145 bash[79878]: cluster 2024-03-07T03:46:55.613815+0000 mgr.y (mgr.34107) 1052 : cluster 0 pgmap v1660: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:46:57.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:57 smithi145 bash[79878]: cluster 2024-03-07T03:46:56.092687+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e984: 8 total, 8 up, 8 in 2024-03-07T03:46:57.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:57 smithi145 bash[79878]: audit 2024-03-07T03:46:56.096413+0000 mon.a (mon.0) 1807 : audit 1 from='client.? 172.21.15.145:0/3983169901' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-44","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:46:57.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:57 smithi145 bash[82329]: cluster 2024-03-07T03:46:55.613815+0000 mgr.y (mgr.34107) 1052 : cluster 0 pgmap v1660: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:46:57.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:57 smithi145 bash[82329]: cluster 2024-03-07T03:46:56.092687+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e984: 8 total, 8 up, 8 in 2024-03-07T03:46:57.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:57 smithi145 bash[82329]: audit 2024-03-07T03:46:56.096413+0000 mon.a (mon.0) 1807 : audit 1 from='client.? 172.21.15.145:0/3983169901' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-44","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:46:57.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:57 smithi189 bash[66969]: cluster 2024-03-07T03:46:55.613815+0000 mgr.y (mgr.34107) 1052 : cluster 0 pgmap v1660: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:46:57.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:57 smithi189 bash[66969]: cluster 2024-03-07T03:46:56.092687+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e984: 8 total, 8 up, 8 in 2024-03-07T03:46:57.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:57 smithi189 bash[66969]: audit 2024-03-07T03:46:56.096413+0000 mon.a (mon.0) 1807 : audit 1 from='client.? 172.21.15.145:0/3983169901' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-44","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:46:58.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:58 smithi145 bash[79878]: audit 2024-03-07T03:46:57.086132+0000 mon.a (mon.0) 1808 : audit 1 from='client.? 172.21.15.145:0/3983169901' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-44","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:46:58.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:58 smithi145 bash[79878]: cluster 2024-03-07T03:46:57.095280+0000 mon.a (mon.0) 1809 : cluster 0 osdmap e985: 8 total, 8 up, 8 in 2024-03-07T03:46:58.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:58 smithi145 bash[82329]: audit 2024-03-07T03:46:57.086132+0000 mon.a (mon.0) 1808 : audit 1 from='client.? 172.21.15.145:0/3983169901' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-44","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:46:58.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:58 smithi145 bash[82329]: cluster 2024-03-07T03:46:57.095280+0000 mon.a (mon.0) 1809 : cluster 0 osdmap e985: 8 total, 8 up, 8 in 2024-03-07T03:46:58.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:58 smithi189 bash[66969]: audit 2024-03-07T03:46:57.086132+0000 mon.a (mon.0) 1808 : audit 1 from='client.? 172.21.15.145:0/3983169901' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-44","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:46:58.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:58 smithi189 bash[66969]: cluster 2024-03-07T03:46:57.095280+0000 mon.a (mon.0) 1809 : cluster 0 osdmap e985: 8 total, 8 up, 8 in 2024-03-07T03:46:59.175 INFO:tasks.workunit.client.0.smithi145.stdout: wrote [865025~96372,972896~38717,4538954~95473,5996460~56558,6486217~8254,10477672~94066,13855918~97322,14934530~5542,16980227~54299,19736649~5393] 2024-03-07T03:46:59.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:59 smithi145 bash[82329]: cluster 2024-03-07T03:46:57.614547+0000 mgr.y (mgr.34107) 1053 : cluster 0 pgmap v1663: 105 pgs: 7 creating+peering, 25 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:46:59.566 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:46:59 smithi145 bash[82329]: cluster 2024-03-07T03:46:58.113946+0000 mon.a (mon.0) 1810 : cluster 0 osdmap e986: 8 total, 8 up, 8 in 2024-03-07T03:46:59.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:59 smithi145 bash[79878]: cluster 2024-03-07T03:46:57.614547+0000 mgr.y (mgr.34107) 1053 : cluster 0 pgmap v1663: 105 pgs: 7 creating+peering, 25 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:46:59.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:46:59 smithi145 bash[79878]: cluster 2024-03-07T03:46:58.113946+0000 mon.a (mon.0) 1810 : cluster 0 osdmap e986: 8 total, 8 up, 8 in 2024-03-07T03:46:59.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:59 smithi189 bash[66969]: cluster 2024-03-07T03:46:57.614547+0000 mgr.y (mgr.34107) 1053 : cluster 0 pgmap v1663: 105 pgs: 7 creating+peering, 25 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:46:59.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:46:59 smithi189 bash[66969]: cluster 2024-03-07T03:46:58.113946+0000 mon.a (mon.0) 1810 : cluster 0 osdmap e986: 8 total, 8 up, 8 in 2024-03-07T03:47:00.170 INFO:tasks.workunit.client.0.smithi145.stdout: wrote [2871387~67665,3500446~93096,3796750~42959,5719086~12901,8871195~74212,10067897~40021,17700727~35519,17883421~45414,19436307~34970] 2024-03-07T03:47:00.176 INFO:tasks.workunit.client.0.smithi145.stdout: diff was [1011613~2828096,5719086~12901,8871195~74212,10067897~40021,17700727~35519,17883421~45414,19436307~34970] 2024-03-07T03:47:00.183 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/0.DiffIterate (5048 ms) 2024-03-07T03:47:00.184 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministic 2024-03-07T03:47:00.184 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:47:00.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:00 smithi145 bash[79878]: cluster 2024-03-07T03:46:59.116431+0000 mon.a (mon.0) 1811 : cluster 0 osdmap e987: 8 total, 8 up, 8 in 2024-03-07T03:47:00.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:00 smithi145 bash[82329]: cluster 2024-03-07T03:46:59.116431+0000 mon.a (mon.0) 1811 : cluster 0 osdmap e987: 8 total, 8 up, 8 in 2024-03-07T03:47:00.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:00 smithi189 bash[66969]: cluster 2024-03-07T03:46:59.116431+0000 mon.a (mon.0) 1811 : cluster 0 osdmap e987: 8 total, 8 up, 8 in 2024-03-07T03:47:01.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:01 smithi145 bash[79878]: cluster 2024-03-07T03:46:59.615234+0000 mgr.y (mgr.34107) 1054 : cluster 0 pgmap v1666: 105 pgs: 7 creating+peering, 12 unknown, 86 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-07T03:47:01.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:01 smithi145 bash[79878]: cluster 2024-03-07T03:47:00.134528+0000 mon.a (mon.0) 1812 : cluster 0 osdmap e988: 8 total, 8 up, 8 in 2024-03-07T03:47:01.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:01 smithi145 bash[82329]: cluster 2024-03-07T03:46:59.615234+0000 mgr.y (mgr.34107) 1054 : cluster 0 pgmap v1666: 105 pgs: 7 creating+peering, 12 unknown, 86 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-07T03:47:01.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:01 smithi145 bash[82329]: cluster 2024-03-07T03:47:00.134528+0000 mon.a (mon.0) 1812 : cluster 0 osdmap e988: 8 total, 8 up, 8 in 2024-03-07T03:47:01.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:01 smithi189 bash[66969]: cluster 2024-03-07T03:46:59.615234+0000 mgr.y (mgr.34107) 1054 : cluster 0 pgmap v1666: 105 pgs: 7 creating+peering, 12 unknown, 86 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-07T03:47:01.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:01 smithi189 bash[66969]: cluster 2024-03-07T03:47:00.134528+0000 mon.a (mon.0) 1812 : cluster 0 osdmap e988: 8 total, 8 up, 8 in 2024-03-07T03:47:02.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:02 smithi145 bash[79878]: cluster 2024-03-07T03:47:01.138009+0000 mon.a (mon.0) 1813 : cluster 0 osdmap e989: 8 total, 8 up, 8 in 2024-03-07T03:47:02.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:02 smithi145 bash[79878]: cluster 2024-03-07T03:47:01.616048+0000 mgr.y (mgr.34107) 1055 : cluster 0 pgmap v1669: 105 pgs: 3 creating+peering, 102 active+clean; 9.4 MiB data, 625 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 179 KiB/s wr, 27 op/s 2024-03-07T03:47:02.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:02 smithi145 bash[82329]: cluster 2024-03-07T03:47:01.138009+0000 mon.a (mon.0) 1813 : cluster 0 osdmap e989: 8 total, 8 up, 8 in 2024-03-07T03:47:02.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:02 smithi145 bash[82329]: cluster 2024-03-07T03:47:01.616048+0000 mgr.y (mgr.34107) 1055 : cluster 0 pgmap v1669: 105 pgs: 3 creating+peering, 102 active+clean; 9.4 MiB data, 625 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 179 KiB/s wr, 27 op/s 2024-03-07T03:47:02.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:02 smithi189 bash[66969]: cluster 2024-03-07T03:47:01.138009+0000 mon.a (mon.0) 1813 : cluster 0 osdmap e989: 8 total, 8 up, 8 in 2024-03-07T03:47:02.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:02 smithi189 bash[66969]: cluster 2024-03-07T03:47:01.616048+0000 mgr.y (mgr.34107) 1055 : cluster 0 pgmap v1669: 105 pgs: 3 creating+peering, 102 active+clean; 9.4 MiB data, 625 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 179 KiB/s wr, 27 op/s 2024-03-07T03:47:02.587 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:47:02 smithi189 bash[59814]: ts=2024-03-07T03:47:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:47:03.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:47:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:47:02] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2024-03-07T03:47:03.216 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministic (3033 ms) 2024-03-07T03:47:03.216 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicPP 2024-03-07T03:47:03.217 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:47:03.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:03 smithi145 bash[79878]: cluster 2024-03-07T03:47:02.155834+0000 mon.a (mon.0) 1814 : cluster 0 osdmap e990: 8 total, 8 up, 8 in 2024-03-07T03:47:03.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:03 smithi145 bash[79878]: audit 2024-03-07T03:47:02.679576+0000 mon.a (mon.0) 1815 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:03.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:03 smithi145 bash[82329]: cluster 2024-03-07T03:47:02.155834+0000 mon.a (mon.0) 1814 : cluster 0 osdmap e990: 8 total, 8 up, 8 in 2024-03-07T03:47:03.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:03 smithi145 bash[82329]: audit 2024-03-07T03:47:02.679576+0000 mon.a (mon.0) 1815 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:03.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:03 smithi189 bash[66969]: cluster 2024-03-07T03:47:02.155834+0000 mon.a (mon.0) 1814 : cluster 0 osdmap e990: 8 total, 8 up, 8 in 2024-03-07T03:47:03.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:03 smithi189 bash[66969]: audit 2024-03-07T03:47:02.679576+0000 mon.a (mon.0) 1815 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:04.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:04 smithi145 bash[79878]: cluster 2024-03-07T03:47:03.169012+0000 mon.a (mon.0) 1816 : cluster 0 osdmap e991: 8 total, 8 up, 8 in 2024-03-07T03:47:04.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:04 smithi145 bash[79878]: cluster 2024-03-07T03:47:03.617014+0000 mgr.y (mgr.34107) 1056 : cluster 0 pgmap v1672: 105 pgs: 105 active+clean; 14 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 259 KiB/s wr, 65 op/s 2024-03-07T03:47:04.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:04 smithi145 bash[82329]: cluster 2024-03-07T03:47:03.169012+0000 mon.a (mon.0) 1816 : cluster 0 osdmap e991: 8 total, 8 up, 8 in 2024-03-07T03:47:04.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:04 smithi145 bash[82329]: cluster 2024-03-07T03:47:03.617014+0000 mgr.y (mgr.34107) 1056 : cluster 0 pgmap v1672: 105 pgs: 105 active+clean; 14 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 259 KiB/s wr, 65 op/s 2024-03-07T03:47:04.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:04 smithi189 bash[66969]: cluster 2024-03-07T03:47:03.169012+0000 mon.a (mon.0) 1816 : cluster 0 osdmap e991: 8 total, 8 up, 8 in 2024-03-07T03:47:04.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:04 smithi189 bash[66969]: cluster 2024-03-07T03:47:03.617014+0000 mgr.y (mgr.34107) 1056 : cluster 0 pgmap v1672: 105 pgs: 105 active+clean; 14 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 259 KiB/s wr, 65 op/s 2024-03-07T03:47:05.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:05 smithi145 bash[79878]: cluster 2024-03-07T03:47:04.181773+0000 mon.a (mon.0) 1817 : cluster 0 osdmap e992: 8 total, 8 up, 8 in 2024-03-07T03:47:05.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:05 smithi145 bash[82329]: cluster 2024-03-07T03:47:04.181773+0000 mon.a (mon.0) 1817 : cluster 0 osdmap e992: 8 total, 8 up, 8 in 2024-03-07T03:47:05.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:05 smithi189 bash[66969]: cluster 2024-03-07T03:47:04.181773+0000 mon.a (mon.0) 1817 : cluster 0 osdmap e992: 8 total, 8 up, 8 in 2024-03-07T03:47:06.258 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicPP (3041 ms) 2024-03-07T03:47:06.258 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDiscard 2024-03-07T03:47:06.258 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:47:06.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:06 smithi145 bash[79878]: cluster 2024-03-07T03:47:05.189788+0000 mon.a (mon.0) 1818 : cluster 0 osdmap e993: 8 total, 8 up, 8 in 2024-03-07T03:47:06.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:06 smithi145 bash[79878]: cluster 2024-03-07T03:47:05.617724+0000 mgr.y (mgr.34107) 1057 : cluster 0 pgmap v1675: 105 pgs: 105 active+clean; 14 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 82 KiB/s wr, 74 op/s 2024-03-07T03:47:06.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:06 smithi145 bash[82329]: cluster 2024-03-07T03:47:05.189788+0000 mon.a (mon.0) 1818 : cluster 0 osdmap e993: 8 total, 8 up, 8 in 2024-03-07T03:47:06.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:06 smithi145 bash[82329]: cluster 2024-03-07T03:47:05.617724+0000 mgr.y (mgr.34107) 1057 : cluster 0 pgmap v1675: 105 pgs: 105 active+clean; 14 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 82 KiB/s wr, 74 op/s 2024-03-07T03:47:06.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:06 smithi189 bash[66969]: cluster 2024-03-07T03:47:05.189788+0000 mon.a (mon.0) 1818 : cluster 0 osdmap e993: 8 total, 8 up, 8 in 2024-03-07T03:47:06.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:06 smithi189 bash[66969]: cluster 2024-03-07T03:47:05.617724+0000 mgr.y (mgr.34107) 1057 : cluster 0 pgmap v1675: 105 pgs: 105 active+clean; 14 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 82 KiB/s wr, 74 op/s 2024-03-07T03:47:07.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:07 smithi145 bash[79878]: cluster 2024-03-07T03:47:06.205106+0000 mon.a (mon.0) 1819 : cluster 0 osdmap e994: 8 total, 8 up, 8 in 2024-03-07T03:47:07.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:07 smithi145 bash[82329]: cluster 2024-03-07T03:47:06.205106+0000 mon.a (mon.0) 1819 : cluster 0 osdmap e994: 8 total, 8 up, 8 in 2024-03-07T03:47:07.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:07 smithi189 bash[66969]: cluster 2024-03-07T03:47:06.205106+0000 mon.a (mon.0) 1819 : cluster 0 osdmap e994: 8 total, 8 up, 8 in 2024-03-07T03:47:08.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:08 smithi145 bash[79878]: cluster 2024-03-07T03:47:07.209591+0000 mon.a (mon.0) 1820 : cluster 0 osdmap e995: 8 total, 8 up, 8 in 2024-03-07T03:47:08.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:08 smithi145 bash[79878]: cluster 2024-03-07T03:47:07.618465+0000 mgr.y (mgr.34107) 1058 : cluster 0 pgmap v1678: 105 pgs: 105 active+clean; 14 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 6.0 KiB/s wr, 60 op/s 2024-03-07T03:47:08.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:08 smithi145 bash[79878]: cluster 2024-03-07T03:47:08.098235+0000 mon.a (mon.0) 1821 : cluster 0 osdmap e996: 8 total, 8 up, 8 in 2024-03-07T03:47:08.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:08 smithi145 bash[82329]: cluster 2024-03-07T03:47:07.209591+0000 mon.a (mon.0) 1820 : cluster 0 osdmap e995: 8 total, 8 up, 8 in 2024-03-07T03:47:08.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:08 smithi145 bash[82329]: cluster 2024-03-07T03:47:07.618465+0000 mgr.y (mgr.34107) 1058 : cluster 0 pgmap v1678: 105 pgs: 105 active+clean; 14 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 6.0 KiB/s wr, 60 op/s 2024-03-07T03:47:08.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:08 smithi145 bash[82329]: cluster 2024-03-07T03:47:08.098235+0000 mon.a (mon.0) 1821 : cluster 0 osdmap e996: 8 total, 8 up, 8 in 2024-03-07T03:47:08.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:08 smithi189 bash[66969]: cluster 2024-03-07T03:47:07.209591+0000 mon.a (mon.0) 1820 : cluster 0 osdmap e995: 8 total, 8 up, 8 in 2024-03-07T03:47:08.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:08 smithi189 bash[66969]: cluster 2024-03-07T03:47:07.618465+0000 mgr.y (mgr.34107) 1058 : cluster 0 pgmap v1678: 105 pgs: 105 active+clean; 14 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 6.0 KiB/s wr, 60 op/s 2024-03-07T03:47:08.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:08 smithi189 bash[66969]: cluster 2024-03-07T03:47:08.098235+0000 mon.a (mon.0) 1821 : cluster 0 osdmap e996: 8 total, 8 up, 8 in 2024-03-07T03:47:09.155 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/0.DiffIterateDiscard (2896 ms) 2024-03-07T03:47:09.156 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStress 2024-03-07T03:47:09.156 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:47:09.294 INFO:tasks.workunit.client.0.smithi145.stdout: i=0 exists [28424801~7483414,69835636~5983323,110444619~4839831,181503969~3427751,207145722~788427,246435847~6696460,262004794~3842914,273380939~8105530,306796441~7271205] wrote [28424801~7483414,69835636~5983323,110444619~4839831,181503969~3427751,207145722~788427,246435847~6696460,262004794~3842914,273380939~8105530,306796441~7271205] 2024-03-07T03:47:10.226 INFO:tasks.workunit.client.0.smithi145.stdout: i=1 exists [7302669~1437061,28424801~7483414,69835636~5983323,110444619~4839831,156381382~6977164,171217122~7466361,178926356~6005364,192552650~6703427,207145722~788427,226998752~6514249,246435847~6696460,262004794~4594643,273380939~8529825,306796441~7271205] wrote [7302669~1437061,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,226998752~6514249,264992561~1606876,279969482~1941282] 2024-03-07T03:47:10.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:10 smithi145 bash[79878]: cluster 2024-03-07T03:47:09.100921+0000 mon.a (mon.0) 1822 : cluster 0 osdmap e997: 8 total, 8 up, 8 in 2024-03-07T03:47:10.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:10 smithi145 bash[79878]: audit 2024-03-07T03:47:09.750450+0000 mon.a (mon.0) 1823 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:10.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:10 smithi145 bash[82329]: cluster 2024-03-07T03:47:09.100921+0000 mon.a (mon.0) 1822 : cluster 0 osdmap e997: 8 total, 8 up, 8 in 2024-03-07T03:47:10.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:10 smithi145 bash[82329]: audit 2024-03-07T03:47:09.750450+0000 mon.a (mon.0) 1823 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:10.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:10 smithi189 bash[66969]: cluster 2024-03-07T03:47:09.100921+0000 mon.a (mon.0) 1822 : cluster 0 osdmap e997: 8 total, 8 up, 8 in 2024-03-07T03:47:10.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:10 smithi189 bash[66969]: audit 2024-03-07T03:47:09.750450+0000 mon.a (mon.0) 1823 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:11.218 INFO:tasks.workunit.client.0.smithi145.stdout: i=2 exists [2318850~3094327,7302669~1437061,28424801~7483414,69835636~5983323,97961740~460363,110444619~4839831,156381382~6977164,171217122~7466361,178926356~6005364,192552650~6703427,207145722~788427,214732945~2402879,226998752~6514249,246435847~6696460,257695447~13880738,273380939~8529825,306796441~7271205,322156674~5263615,338329190~3592500,379305389~6030299] wrote [2318850~3094327,97961740~460363,214732945~2402879,247013430~638075,257695447~6054536,264661926~6914259,322156674~5263615,338329190~3592500,379305389~6030299] 2024-03-07T03:47:11.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:11 smithi145 bash[79878]: cluster 2024-03-07T03:47:09.619112+0000 mgr.y (mgr.34107) 1059 : cluster 0 pgmap v1681: 105 pgs: 105 active+clean; 14 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 8.2 KiB/s wr, 84 op/s 2024-03-07T03:47:11.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:11 smithi145 bash[79878]: cluster 2024-03-07T03:47:10.109250+0000 mon.a (mon.0) 1824 : cluster 0 osdmap e998: 8 total, 8 up, 8 in 2024-03-07T03:47:11.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:11 smithi145 bash[82329]: cluster 2024-03-07T03:47:09.619112+0000 mgr.y (mgr.34107) 1059 : cluster 0 pgmap v1681: 105 pgs: 105 active+clean; 14 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 8.2 KiB/s wr, 84 op/s 2024-03-07T03:47:11.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:11 smithi145 bash[82329]: cluster 2024-03-07T03:47:10.109250+0000 mon.a (mon.0) 1824 : cluster 0 osdmap e998: 8 total, 8 up, 8 in 2024-03-07T03:47:11.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:11 smithi189 bash[66969]: cluster 2024-03-07T03:47:09.619112+0000 mgr.y (mgr.34107) 1059 : cluster 0 pgmap v1681: 105 pgs: 105 active+clean; 14 MiB data, 628 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 8.2 KiB/s wr, 84 op/s 2024-03-07T03:47:11.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:11 smithi189 bash[66969]: cluster 2024-03-07T03:47:10.109250+0000 mon.a (mon.0) 1824 : cluster 0 osdmap e998: 8 total, 8 up, 8 in 2024-03-07T03:47:12.259 INFO:tasks.workunit.client.0.smithi145.stdout: i=3 exists [2318850~3094327,7302669~1437061,28424801~7483414,36454755~3492606,60790717~5974615,69531456~6287503,87852947~7066102,97961740~460363,110444619~4839831,156381382~6977164,171217122~7466361,178926356~6005364,192552650~6703427,207145722~788427,214732945~2402879,226998752~6514249,246435847~6696460,257695447~13880738,273380939~10525372,306796441~7271205,322156674~5263615,331086294~10835396,353776387~5206630,364517326~7139856,379305389~6030299] wrote [36454755~3492606,60790717~5974615,69531456~1516890,87852947~7066102,280033366~3872945,331086294~8284072,353776387~5206630,364517326~7139856] 2024-03-07T03:47:12.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:12 smithi145 bash[79878]: cluster 2024-03-07T03:47:11.125263+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e999: 8 total, 8 up, 8 in 2024-03-07T03:47:12.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:12 smithi145 bash[82329]: cluster 2024-03-07T03:47:11.125263+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e999: 8 total, 8 up, 8 in 2024-03-07T03:47:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:12 smithi189 bash[66969]: cluster 2024-03-07T03:47:11.125263+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e999: 8 total, 8 up, 8 in 2024-03-07T03:47:12.587 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:47:12 smithi189 bash[59814]: ts=2024-03-07T03:47:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:47:13.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:47:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:47:12] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2024-03-07T03:47:13.238 INFO:tasks.workunit.client.0.smithi145.stdout: i=4 exists [2318850~3094327,7302669~1437061,26015034~59803,28424801~7483414,36454755~3492606,60790717~5974615,69531456~6287503,87852947~7066102,97961740~460363,102460011~15846829,156381382~6977164,171217122~7466361,178926356~6005364,192552650~6703427,207145722~788427,214732945~2402879,226998752~6514249,245529155~26047030,273380939~10525372,284525397~4534989,306796441~7271205,322156674~5263615,329627709~325719,331086294~15243300,353776387~5206630,364517326~7139856,379305389~6030299] wrote [26015034~59803,102460011~8177373,111781168~6525672,245529155~12485317,284525397~4534989,329627709~325719,338478793~2173931,341629853~4699741] 2024-03-07T03:47:13.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:13 smithi145 bash[79878]: cluster 2024-03-07T03:47:11.619928+0000 mgr.y (mgr.34107) 1060 : cluster 0 pgmap v1684: 105 pgs: 105 active+clean; 50 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 5.6 MiB/s wr, 108 op/s 2024-03-07T03:47:13.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:13 smithi145 bash[79878]: cluster 2024-03-07T03:47:12.136459+0000 mon.a (mon.0) 1826 : cluster 0 osdmap e1000: 8 total, 8 up, 8 in 2024-03-07T03:47:13.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:13 smithi145 bash[82329]: cluster 2024-03-07T03:47:11.619928+0000 mgr.y (mgr.34107) 1060 : cluster 0 pgmap v1684: 105 pgs: 105 active+clean; 50 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 5.6 MiB/s wr, 108 op/s 2024-03-07T03:47:13.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:13 smithi145 bash[82329]: cluster 2024-03-07T03:47:12.136459+0000 mon.a (mon.0) 1826 : cluster 0 osdmap e1000: 8 total, 8 up, 8 in 2024-03-07T03:47:13.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:13 smithi189 bash[66969]: cluster 2024-03-07T03:47:11.619928+0000 mgr.y (mgr.34107) 1060 : cluster 0 pgmap v1684: 105 pgs: 105 active+clean; 50 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 5.6 MiB/s wr, 108 op/s 2024-03-07T03:47:13.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:13 smithi189 bash[66969]: cluster 2024-03-07T03:47:12.136459+0000 mon.a (mon.0) 1826 : cluster 0 osdmap e1000: 8 total, 8 up, 8 in 2024-03-07T03:47:14.294 INFO:tasks.workunit.client.0.smithi145.stdout: i=5 exists [2318850~3698821,7302669~1437061,26015034~59803,28424801~7483414,36454755~3492606,43820204~645623,60790717~5974615,69531456~6287503,84515256~10403793,97961740~460363,102460011~15846829,128110794~4019853,156381382~6977164,171217122~7466361,178926356~6005364,192552650~6703427,207145722~788427,214732945~2402879,226998752~6514249,245529155~26047030,273380939~10525372,284525397~4534989,306796441~7271205,322156674~5263615,329627709~325719,331086294~17095149,353776387~5206630,362225914~9431268,379305389~6030299] wrote [4366544~1651127,43820204~645623,84515256~7087146,128110794~4019853,334175736~6822610,342998383~5183060,362225914~7600999] 2024-03-07T03:47:14.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:14 smithi145 bash[79878]: cluster 2024-03-07T03:47:13.149248+0000 mon.a (mon.0) 1827 : cluster 0 osdmap e1001: 8 total, 8 up, 8 in 2024-03-07T03:47:14.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:14 smithi145 bash[79878]: cluster 2024-03-07T03:47:13.620878+0000 mgr.y (mgr.34107) 1061 : cluster 0 pgmap v1687: 105 pgs: 105 active+clean; 175 MiB data, 945 MiB used, 714 GiB / 715 GiB avail; 148 KiB/s rd, 33 MiB/s wr, 262 op/s 2024-03-07T03:47:14.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:14 smithi145 bash[82329]: cluster 2024-03-07T03:47:13.149248+0000 mon.a (mon.0) 1827 : cluster 0 osdmap e1001: 8 total, 8 up, 8 in 2024-03-07T03:47:14.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:14 smithi145 bash[82329]: cluster 2024-03-07T03:47:13.620878+0000 mgr.y (mgr.34107) 1061 : cluster 0 pgmap v1687: 105 pgs: 105 active+clean; 175 MiB data, 945 MiB used, 714 GiB / 715 GiB avail; 148 KiB/s rd, 33 MiB/s wr, 262 op/s 2024-03-07T03:47:14.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:14 smithi189 bash[66969]: cluster 2024-03-07T03:47:13.149248+0000 mon.a (mon.0) 1827 : cluster 0 osdmap e1001: 8 total, 8 up, 8 in 2024-03-07T03:47:14.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:14 smithi189 bash[66969]: cluster 2024-03-07T03:47:13.620878+0000 mgr.y (mgr.34107) 1061 : cluster 0 pgmap v1687: 105 pgs: 105 active+clean; 175 MiB data, 945 MiB used, 714 GiB / 715 GiB avail; 148 KiB/s rd, 33 MiB/s wr, 262 op/s 2024-03-07T03:47:15.271 INFO:tasks.workunit.client.0.smithi145.stdout: i=6 exists [2318850~7397726,13425281~5376438,26015034~59803,28424801~11522560,43820204~645623,60790717~5974615,69531456~6287503,84515256~10403793,95397413~180359,97961740~460363,102460011~15846829,126185278~6228220,156381382~6977164,171217122~7466361,178926356~6005364,192552650~6703427,207145722~788427,214732945~2402879,223103880~1116779,226998752~6514249,245529155~26047030,273380939~10525372,284525397~4534989,304833665~9233981,322156674~5263615,329627709~325719,331086294~17095149,348725436~10257581,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] wrote [4327490~5389086,13425281~5376438,31786956~5603854,95397413~180359,126185278~6228220,223103880~1116779,304833665~5535071,348725436~6267586,395654001~4615410,405400684~7492719] 2024-03-07T03:47:15.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:15 smithi145 bash[79878]: cluster 2024-03-07T03:47:14.157603+0000 mon.a (mon.0) 1828 : cluster 0 osdmap e1002: 8 total, 8 up, 8 in 2024-03-07T03:47:15.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:15 smithi145 bash[79878]: audit 2024-03-07T03:47:15.032395+0000 mon.a (mon.0) 1829 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:47:15.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:15 smithi145 bash[82329]: cluster 2024-03-07T03:47:14.157603+0000 mon.a (mon.0) 1828 : cluster 0 osdmap e1002: 8 total, 8 up, 8 in 2024-03-07T03:47:15.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:15 smithi145 bash[82329]: audit 2024-03-07T03:47:15.032395+0000 mon.a (mon.0) 1829 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:47:15.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:15 smithi189 bash[66969]: cluster 2024-03-07T03:47:14.157603+0000 mon.a (mon.0) 1828 : cluster 0 osdmap e1002: 8 total, 8 up, 8 in 2024-03-07T03:47:15.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:15 smithi189 bash[66969]: audit 2024-03-07T03:47:15.032395+0000 mon.a (mon.0) 1829 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:47:16.286 INFO:tasks.workunit.client.0.smithi145.stdout: i=7 exists [2318850~16482869,26015034~59803,28424801~13204497,43820204~645623,52568246~3756564,60790717~5974615,69531456~6287503,84515256~12403044,97961740~460363,102460011~20004800,126185278~6228220,135729567~5523291,147158925~1058372,156381382~6977164,171217122~7466361,178926356~6005364,192552650~6703427,207145722~788427,214732945~2402879,223103880~1116779,226998752~6514249,243436846~28139339,273380939~10525372,284525397~4534989,304833665~9233981,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] wrote [8700349~6073181,36920789~4708509,52568246~3756564,94820503~2097797,116976006~5488805,135729567~5523291,147158925~1058372,243436846~7263777,356430482~4952074] 2024-03-07T03:47:16.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:16 smithi145 bash[79878]: cluster 2024-03-07T03:47:15.164561+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e1003: 8 total, 8 up, 8 in 2024-03-07T03:47:16.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:16 smithi145 bash[79878]: cluster 2024-03-07T03:47:15.621533+0000 mgr.y (mgr.34107) 1062 : cluster 0 pgmap v1690: 105 pgs: 105 active+clean; 200 MiB data, 1010 MiB used, 714 GiB / 715 GiB avail; 169 KiB/s rd, 34 MiB/s wr, 272 op/s 2024-03-07T03:47:16.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:16 smithi145 bash[82329]: cluster 2024-03-07T03:47:15.164561+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e1003: 8 total, 8 up, 8 in 2024-03-07T03:47:16.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:16 smithi145 bash[82329]: cluster 2024-03-07T03:47:15.621533+0000 mgr.y (mgr.34107) 1062 : cluster 0 pgmap v1690: 105 pgs: 105 active+clean; 200 MiB data, 1010 MiB used, 714 GiB / 715 GiB avail; 169 KiB/s rd, 34 MiB/s wr, 272 op/s 2024-03-07T03:47:16.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:16 smithi189 bash[66969]: cluster 2024-03-07T03:47:15.164561+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e1003: 8 total, 8 up, 8 in 2024-03-07T03:47:16.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:16 smithi189 bash[66969]: cluster 2024-03-07T03:47:15.621533+0000 mgr.y (mgr.34107) 1062 : cluster 0 pgmap v1690: 105 pgs: 105 active+clean; 200 MiB data, 1010 MiB used, 714 GiB / 715 GiB avail; 169 KiB/s rd, 34 MiB/s wr, 272 op/s 2024-03-07T03:47:17.268 INFO:tasks.workunit.client.0.smithi145.stdout: i=8 exists [2318850~16482869,26015034~59803,28424801~14409296,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,69531456~6859880,84405198~12513102,97961740~460363,102460011~20004800,126185278~6228220,135729567~5523291,147158925~1058372,156381382~11790402,171217122~7466361,178926356~6005364,190129935~302950,192552650~6703427,207145722~788427,214732945~2761071,223103880~1116779,226998752~6514249,243436846~28139339,273380939~10525372,284525397~4534989,304833665~9610132,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] wrote [38508433~4325664,47609853~1989575,71963918~4427418,84405198~8165644,163070735~5101049,190129935~302950,216500076~993940,256893024~361554,262307318~1813905,311853868~2589929] 2024-03-07T03:47:17.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:17 smithi145 bash[79878]: cluster 2024-03-07T03:47:16.181554+0000 mon.a (mon.0) 1831 : cluster 0 osdmap e1004: 8 total, 8 up, 8 in 2024-03-07T03:47:17.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:17 smithi145 bash[82329]: cluster 2024-03-07T03:47:16.181554+0000 mon.a (mon.0) 1831 : cluster 0 osdmap e1004: 8 total, 8 up, 8 in 2024-03-07T03:47:17.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:17 smithi189 bash[66969]: cluster 2024-03-07T03:47:16.181554+0000 mon.a (mon.0) 1831 : cluster 0 osdmap e1004: 8 total, 8 up, 8 in 2024-03-07T03:47:18.312 INFO:tasks.workunit.client.0.smithi145.stdout: i=9 exists [2318850~16482869,26015034~59803,28424801~14958017,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,67520406~8870930,84405198~12513102,97961740~460363,102460011~20004800,126185278~6228220,135729567~5523291,147158925~1058372,156381382~11790402,171217122~7466361,178926356~9707135,190129935~302950,192552650~6703427,207145722~788427,214732945~2761071,223103880~1116779,226998752~6514249,243436846~28139339,273380939~10525372,284525397~4534989,304833665~9610132,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,402337569~10555834] wrote [32517841~6046506,42356129~1026689,67520406~8127436,180869607~7763884,224056275~91427,227584913~2698758,356226413~335101,402337569~8017381] 2024-03-07T03:47:18.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:18 smithi145 bash[79878]: cluster 2024-03-07T03:47:17.178382+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e1005: 8 total, 8 up, 8 in 2024-03-07T03:47:18.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:18 smithi145 bash[79878]: cluster 2024-03-07T03:47:17.622198+0000 mgr.y (mgr.34107) 1063 : cluster 0 pgmap v1693: 105 pgs: 105 active+clean; 323 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 33 MiB/s wr, 71 op/s 2024-03-07T03:47:18.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:18 smithi145 bash[79878]: audit 2024-03-07T03:47:17.679992+0000 mon.a (mon.0) 1833 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:18.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:18 smithi145 bash[82329]: cluster 2024-03-07T03:47:17.178382+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e1005: 8 total, 8 up, 8 in 2024-03-07T03:47:18.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:18 smithi145 bash[82329]: cluster 2024-03-07T03:47:17.622198+0000 mgr.y (mgr.34107) 1063 : cluster 0 pgmap v1693: 105 pgs: 105 active+clean; 323 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 33 MiB/s wr, 71 op/s 2024-03-07T03:47:18.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:18 smithi145 bash[82329]: audit 2024-03-07T03:47:17.679992+0000 mon.a (mon.0) 1833 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:18.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:18 smithi189 bash[66969]: cluster 2024-03-07T03:47:17.178382+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e1005: 8 total, 8 up, 8 in 2024-03-07T03:47:18.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:18 smithi189 bash[66969]: cluster 2024-03-07T03:47:17.622198+0000 mgr.y (mgr.34107) 1063 : cluster 0 pgmap v1693: 105 pgs: 105 active+clean; 323 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 33 MiB/s wr, 71 op/s 2024-03-07T03:47:18.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:18 smithi189 bash[66969]: audit 2024-03-07T03:47:17.679992+0000 mon.a (mon.0) 1833 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:19.321 INFO:tasks.workunit.client.0.smithi145.stdout: i=10 exists [2318850~16482869,26015034~59803,28424801~14958017,43820204~645623,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~20085405,126185278~6228220,135729567~5523291,147158925~1058372,156381382~11790402,171217122~7466361,178926356~9707135,190129935~302950,192552650~6703427,202024823~1392556,207097364~4743737,214732945~2761071,223103880~1116779,226998752~6514249,243436846~28139339,273380939~10525372,284525397~7006898,304833665~9610132,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,395654001~4615410,402337569~10555834] wrote [11633950~3602920,62513806~6402641,89585825~97099,104659775~4064028,114533580~8011836,202024823~1392556,207097364~4743737,285480198~6052097,358406858~1989264,375226132~5321458] 2024-03-07T03:47:19.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:19 smithi145 bash[79878]: cluster 2024-03-07T03:47:18.201669+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e1006: 8 total, 8 up, 8 in 2024-03-07T03:47:19.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:19 smithi145 bash[82329]: cluster 2024-03-07T03:47:18.201669+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e1006: 8 total, 8 up, 8 in 2024-03-07T03:47:19.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:19 smithi189 bash[66969]: cluster 2024-03-07T03:47:18.201669+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e1006: 8 total, 8 up, 8 in 2024-03-07T03:47:20.287 INFO:tasks.workunit.client.0.smithi145.stdout: i=11 exists [2318850~16482869,26015034~59803,28424801~18520750,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~20085405,126185278~6228220,135729567~5523291,147158925~1058372,156128498~12043286,171217122~7466361,178926356~9707135,190129935~302950,192552650~6703427,202024823~1392556,206070966~7558505,214732945~2761071,223103880~1116779,226998752~6514249,243436846~28139339,273380939~10525372,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,395654001~4615410,402337569~10555834] wrote [42576538~4369013,156128498~1778066,206070966~7558505,287416788~2430416,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,404359095~5485689] 2024-03-07T03:47:20.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:20 smithi145 bash[79878]: cluster 2024-03-07T03:47:19.210022+0000 mon.a (mon.0) 1835 : cluster 0 osdmap e1007: 8 total, 8 up, 8 in 2024-03-07T03:47:20.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:20 smithi145 bash[79878]: cluster 2024-03-07T03:47:19.623108+0000 mgr.y (mgr.34107) 1064 : cluster 0 pgmap v1696: 105 pgs: 105 active+clean; 413 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 49 MiB/s wr, 199 op/s 2024-03-07T03:47:20.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:20 smithi145 bash[82329]: cluster 2024-03-07T03:47:19.210022+0000 mon.a (mon.0) 1835 : cluster 0 osdmap e1007: 8 total, 8 up, 8 in 2024-03-07T03:47:20.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:20 smithi145 bash[82329]: cluster 2024-03-07T03:47:19.623108+0000 mgr.y (mgr.34107) 1064 : cluster 0 pgmap v1696: 105 pgs: 105 active+clean; 413 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 49 MiB/s wr, 199 op/s 2024-03-07T03:47:20.586 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:47:20 smithi189 bash[49584]: logger=cleanup t=2024-03-07T03:47:20.194484396Z level=info msg="Completed cleanup jobs" duration=45.366119ms 2024-03-07T03:47:20.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:20 smithi189 bash[66969]: cluster 2024-03-07T03:47:19.210022+0000 mon.a (mon.0) 1835 : cluster 0 osdmap e1007: 8 total, 8 up, 8 in 2024-03-07T03:47:20.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:20 smithi189 bash[66969]: cluster 2024-03-07T03:47:19.623108+0000 mgr.y (mgr.34107) 1064 : cluster 0 pgmap v1696: 105 pgs: 105 active+clean; 413 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 49 MiB/s wr, 199 op/s 2024-03-07T03:47:21.329 INFO:tasks.workunit.client.0.smithi145.stdout: i=12 exists [2318850~16482869,24990107~1581751,28424801~18520750,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~20085405,126185278~6228220,134570734~6682124,147158925~7285607,156128498~12043286,171217122~7466361,178926356~9707135,190129935~302950,192552650~6703427,202024823~1392556,206070966~7558505,214732945~2761071,223103880~1116779,226998752~6514249,243436846~28139339,273380939~10525372,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,395654001~4615410,401472293~11421110] wrote [24990107~1581751,41028236~1102842,68299100~7174136,104516662~7567202,114766453~7272045,134570734~3387789,147989099~6455433,378511893~3011197,390408721~3258158,401472293~6686420] 2024-03-07T03:47:21.566 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:21 smithi145 bash[79878]: cluster 2024-03-07T03:47:20.212595+0000 mon.a (mon.0) 1836 : cluster 0 osdmap e1008: 8 total, 8 up, 8 in 2024-03-07T03:47:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:21 smithi145 bash[79878]: audit 2024-03-07T03:47:20.927591+0000 mon.a (mon.0) 1837 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:21.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:21 smithi145 bash[79878]: audit 2024-03-07T03:47:20.936832+0000 mon.a (mon.0) 1838 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:21.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:21 smithi145 bash[82329]: cluster 2024-03-07T03:47:20.212595+0000 mon.a (mon.0) 1836 : cluster 0 osdmap e1008: 8 total, 8 up, 8 in 2024-03-07T03:47:21.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:21 smithi145 bash[82329]: audit 2024-03-07T03:47:20.927591+0000 mon.a (mon.0) 1837 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:21.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:21 smithi145 bash[82329]: audit 2024-03-07T03:47:20.936832+0000 mon.a (mon.0) 1838 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:21.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:21 smithi189 bash[66969]: cluster 2024-03-07T03:47:20.212595+0000 mon.a (mon.0) 1836 : cluster 0 osdmap e1008: 8 total, 8 up, 8 in 2024-03-07T03:47:21.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:21 smithi189 bash[66969]: audit 2024-03-07T03:47:20.927591+0000 mon.a (mon.0) 1837 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:21.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:21 smithi189 bash[66969]: audit 2024-03-07T03:47:20.936832+0000 mon.a (mon.0) 1838 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:22.364 INFO:tasks.workunit.client.0.smithi145.stdout: i=13 exists [2318850~16482869,24990107~1581751,28424801~18520750,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6228220,134570734~6682124,147158925~7285607,156128498~12043286,171217122~7466361,178926356~9707135,190129935~302950,192552650~10864729,206070966~7558505,214732945~2761071,223103880~1116779,226998752~6514249,243436846~28139339,273380939~10525372,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] wrote [64158202~4966057,82227694~6384327,96152654~1180560,98329964~7380989,117334964~4480713,198381649~4662106,251991784~6059300,259345944~6967198,394824008~6549491] 2024-03-07T03:47:22.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[79878]: cluster 2024-03-07T03:47:21.226474+0000 mon.a (mon.0) 1839 : cluster 0 osdmap e1009: 8 total, 8 up, 8 in 2024-03-07T03:47:22.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[79878]: audit 2024-03-07T03:47:21.237670+0000 mon.a (mon.0) 1840 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:22.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[79878]: audit 2024-03-07T03:47:21.250410+0000 mon.a (mon.0) 1841 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:22.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[79878]: cluster 2024-03-07T03:47:21.624069+0000 mgr.y (mgr.34107) 1065 : cluster 0 pgmap v1699: 105 pgs: 105 active+clean; 500 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 185 KiB/s rd, 44 MiB/s wr, 298 op/s 2024-03-07T03:47:22.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[79878]: audit 2024-03-07T03:47:21.880448+0000 mon.a (mon.0) 1842 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:47:22.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[79878]: audit 2024-03-07T03:47:21.881754+0000 mon.a (mon.0) 1843 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:47:22.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[79878]: audit 2024-03-07T03:47:21.891271+0000 mon.a (mon.0) 1844 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:22.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[82329]: cluster 2024-03-07T03:47:21.226474+0000 mon.a (mon.0) 1839 : cluster 0 osdmap e1009: 8 total, 8 up, 8 in 2024-03-07T03:47:22.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[82329]: audit 2024-03-07T03:47:21.237670+0000 mon.a (mon.0) 1840 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:22.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[82329]: audit 2024-03-07T03:47:21.250410+0000 mon.a (mon.0) 1841 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:22.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[82329]: cluster 2024-03-07T03:47:21.624069+0000 mgr.y (mgr.34107) 1065 : cluster 0 pgmap v1699: 105 pgs: 105 active+clean; 500 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 185 KiB/s rd, 44 MiB/s wr, 298 op/s 2024-03-07T03:47:22.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[82329]: audit 2024-03-07T03:47:21.880448+0000 mon.a (mon.0) 1842 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:47:22.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[82329]: audit 2024-03-07T03:47:21.881754+0000 mon.a (mon.0) 1843 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:47:22.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[82329]: audit 2024-03-07T03:47:21.891271+0000 mon.a (mon.0) 1844 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:22.587 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:47:22 smithi189 bash[59814]: ts=2024-03-07T03:47:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:47:22.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:22 smithi189 bash[66969]: cluster 2024-03-07T03:47:21.226474+0000 mon.a (mon.0) 1839 : cluster 0 osdmap e1009: 8 total, 8 up, 8 in 2024-03-07T03:47:22.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:22 smithi189 bash[66969]: audit 2024-03-07T03:47:21.237670+0000 mon.a (mon.0) 1840 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:22.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:22 smithi189 bash[66969]: audit 2024-03-07T03:47:21.250410+0000 mon.a (mon.0) 1841 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:22.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:22 smithi189 bash[66969]: cluster 2024-03-07T03:47:21.624069+0000 mgr.y (mgr.34107) 1065 : cluster 0 pgmap v1699: 105 pgs: 105 active+clean; 500 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 185 KiB/s rd, 44 MiB/s wr, 298 op/s 2024-03-07T03:47:22.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:22 smithi189 bash[66969]: audit 2024-03-07T03:47:21.880448+0000 mon.a (mon.0) 1842 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:47:22.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:22 smithi189 bash[66969]: audit 2024-03-07T03:47:21.881754+0000 mon.a (mon.0) 1843 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:47:22.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:22 smithi189 bash[66969]: audit 2024-03-07T03:47:21.891271+0000 mon.a (mon.0) 1844 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:47:23.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:47:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:47:22] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.43.0" 2024-03-07T03:47:23.389 INFO:tasks.workunit.client.0.smithi145.stdout: i=14 exists [2318850~16482869,24990107~1581751,27711596~19233955,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6228220,134570734~7674958,145598567~33084916,178926356~9707135,190129935~302950,192552650~10864729,206070966~7558505,214732945~2761071,223103880~1116779,226998752~6514249,243436846~40469465,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] wrote [27711596~2702724,42157137~1900013,84122472~5882764,137121160~5124532,145598567~2134643,151434756~8115093,163152484~13031193,269734316~3905873] 2024-03-07T03:47:23.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:23 smithi145 bash[79878]: cluster 2024-03-07T03:47:22.260918+0000 mon.a (mon.0) 1845 : cluster 0 osdmap e1010: 8 total, 8 up, 8 in 2024-03-07T03:47:23.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:23 smithi145 bash[82329]: cluster 2024-03-07T03:47:22.260918+0000 mon.a (mon.0) 1845 : cluster 0 osdmap e1010: 8 total, 8 up, 8 in 2024-03-07T03:47:23.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:23 smithi189 bash[66969]: cluster 2024-03-07T03:47:22.260918+0000 mon.a (mon.0) 1845 : cluster 0 osdmap e1010: 8 total, 8 up, 8 in 2024-03-07T03:47:24.435 INFO:tasks.workunit.client.0.smithi145.stdout: i=15 exists [2318850~16482869,24990107~1581751,27711596~19233955,47609853~1989575,52484686~6706282,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~33084916,178926356~9707135,190129935~302950,192552650~10864729,206070966~7558505,214732945~2761071,223103880~1116779,226998752~11399638,239142063~44764248,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] wrote [36221741~8159531,52484686~6706282,61973630~5522256,132162650~358865,231680022~6718368,239142063~7324550,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248] 2024-03-07T03:47:24.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:24 smithi145 bash[79878]: cluster 2024-03-07T03:47:23.276242+0000 mon.a (mon.0) 1846 : cluster 0 osdmap e1011: 8 total, 8 up, 8 in 2024-03-07T03:47:24.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:24 smithi145 bash[79878]: cluster 2024-03-07T03:47:23.625003+0000 mgr.y (mgr.34107) 1066 : cluster 0 pgmap v1702: 105 pgs: 105 active+clean; 617 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 51 MiB/s wr, 325 op/s 2024-03-07T03:47:24.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:24 smithi145 bash[82329]: cluster 2024-03-07T03:47:23.276242+0000 mon.a (mon.0) 1846 : cluster 0 osdmap e1011: 8 total, 8 up, 8 in 2024-03-07T03:47:24.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:24 smithi145 bash[82329]: cluster 2024-03-07T03:47:23.625003+0000 mgr.y (mgr.34107) 1066 : cluster 0 pgmap v1702: 105 pgs: 105 active+clean; 617 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 51 MiB/s wr, 325 op/s 2024-03-07T03:47:24.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:24 smithi189 bash[66969]: cluster 2024-03-07T03:47:23.276242+0000 mon.a (mon.0) 1846 : cluster 0 osdmap e1011: 8 total, 8 up, 8 in 2024-03-07T03:47:24.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:24 smithi189 bash[66969]: cluster 2024-03-07T03:47:23.625003+0000 mgr.y (mgr.34107) 1066 : cluster 0 pgmap v1702: 105 pgs: 105 active+clean; 617 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 51 MiB/s wr, 325 op/s 2024-03-07T03:47:25.398 INFO:tasks.workunit.client.0.smithi145.stdout: i=16 exists [2318850~16482869,24990107~1581751,27711596~22230925,52484686~6706282,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~43034924,190129935~302950,192552650~10864729,206070966~7558505,214732945~2761071,223103880~1116779,226998752~11399638,239142063~44764248,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,389185474~12188025,401472293~11421110] wrote [45945810~3996711,69970383~2374636,92768282~4310745,102928466~1050357,175903570~4995247,200983077~344966,270019191~3346561,389185474~6128674,405442075~6643934] 2024-03-07T03:47:25.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:25 smithi145 bash[79878]: cluster 2024-03-07T03:47:24.286325+0000 mon.a (mon.0) 1847 : cluster 0 osdmap e1012: 8 total, 8 up, 8 in 2024-03-07T03:47:25.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:25 smithi145 bash[82329]: cluster 2024-03-07T03:47:24.286325+0000 mon.a (mon.0) 1847 : cluster 0 osdmap e1012: 8 total, 8 up, 8 in 2024-03-07T03:47:25.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:25 smithi189 bash[66969]: cluster 2024-03-07T03:47:24.286325+0000 mon.a (mon.0) 1847 : cluster 0 osdmap e1012: 8 total, 8 up, 8 in 2024-03-07T03:47:26.393 INFO:tasks.workunit.client.0.smithi145.stdout: i=17 exists [2318850~16482869,24990107~1581751,27711596~22230925,52484686~6706282,60790717~15600619,77508972~1810144,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214732945~7988617,223103880~1116779,226998752~11399638,239142063~44764248,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~30296262,362225914~9431268,375226132~10109556,389185474~12188025,401472293~11421110] wrote [43086952~4169818,64033340~3690644,77508972~1810144,91110716~555941,135670601~6428309,186173555~1552003,191170311~4467874,215593174~7128388,346163747~4187506,355622721~665560] 2024-03-07T03:47:26.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:26 smithi145 bash[79878]: cluster 2024-03-07T03:47:25.293056+0000 mon.a (mon.0) 1848 : cluster 0 osdmap e1013: 8 total, 8 up, 8 in 2024-03-07T03:47:26.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:26 smithi145 bash[79878]: cluster 2024-03-07T03:47:25.625724+0000 mgr.y (mgr.34107) 1067 : cluster 0 pgmap v1705: 105 pgs: 105 active+clean; 635 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 233 KiB/s rd, 34 MiB/s wr, 359 op/s 2024-03-07T03:47:26.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:26 smithi145 bash[82329]: cluster 2024-03-07T03:47:25.293056+0000 mon.a (mon.0) 1848 : cluster 0 osdmap e1013: 8 total, 8 up, 8 in 2024-03-07T03:47:26.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:26 smithi145 bash[82329]: cluster 2024-03-07T03:47:25.625724+0000 mgr.y (mgr.34107) 1067 : cluster 0 pgmap v1705: 105 pgs: 105 active+clean; 635 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 233 KiB/s rd, 34 MiB/s wr, 359 op/s 2024-03-07T03:47:26.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:26 smithi189 bash[66969]: cluster 2024-03-07T03:47:25.293056+0000 mon.a (mon.0) 1848 : cluster 0 osdmap e1013: 8 total, 8 up, 8 in 2024-03-07T03:47:26.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:26 smithi189 bash[66969]: cluster 2024-03-07T03:47:25.625724+0000 mgr.y (mgr.34107) 1067 : cluster 0 pgmap v1705: 105 pgs: 105 active+clean; 635 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 233 KiB/s rd, 34 MiB/s wr, 359 op/s 2024-03-07T03:47:27.444 INFO:tasks.workunit.client.0.smithi145.stdout: i=18 exists [2318850~16482869,24990107~1581751,27711596~32337925,60790717~15600619,77508972~1810144,82227694~15105520,97961740~25381762,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214732945~7988617,223103880~1116779,226998752~11399638,239142063~44764248,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~10109556,389185474~23707929] wrote [30382848~5397162,49337602~10711919,120440526~2902976,129286528~6266616,183443572~4858052,309788973~14970654,364013514~6675474,400236992~3711221] 2024-03-07T03:47:27.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:27 smithi145 bash[79878]: cluster 2024-03-07T03:47:26.308817+0000 mon.a (mon.0) 1849 : cluster 0 osdmap e1014: 8 total, 8 up, 8 in 2024-03-07T03:47:27.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:27 smithi145 bash[82329]: cluster 2024-03-07T03:47:26.308817+0000 mon.a (mon.0) 1849 : cluster 0 osdmap e1014: 8 total, 8 up, 8 in 2024-03-07T03:47:27.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:27 smithi189 bash[66969]: cluster 2024-03-07T03:47:26.308817+0000 mon.a (mon.0) 1849 : cluster 0 osdmap e1014: 8 total, 8 up, 8 in 2024-03-07T03:47:28.454 INFO:tasks.workunit.client.0.smithi145.stdout: i=19 exists [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~44764248,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] wrote [75029159~4365000,121402667~3683085,130074465~2050454,214392323~7451196,251105401~1365052,262420310~3474447,292025710~6016132,384690666~8128019,405654356~2651569] 2024-03-07T03:47:28.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:28 smithi189 bash[66969]: cluster 2024-03-07T03:47:27.306183+0000 mon.a (mon.0) 1850 : cluster 0 osdmap e1015: 8 total, 8 up, 8 in 2024-03-07T03:47:28.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:28 smithi189 bash[66969]: cluster 2024-03-07T03:47:27.626450+0000 mgr.y (mgr.34107) 1068 : cluster 0 pgmap v1708: 105 pgs: 105 active+clean; 758 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 33 MiB/s wr, 160 op/s 2024-03-07T03:47:28.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:28 smithi145 bash[79878]: cluster 2024-03-07T03:47:27.306183+0000 mon.a (mon.0) 1850 : cluster 0 osdmap e1015: 8 total, 8 up, 8 in 2024-03-07T03:47:28.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:28 smithi145 bash[79878]: cluster 2024-03-07T03:47:27.626450+0000 mgr.y (mgr.34107) 1068 : cluster 0 pgmap v1708: 105 pgs: 105 active+clean; 758 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 33 MiB/s wr, 160 op/s 2024-03-07T03:47:28.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:28 smithi145 bash[82329]: cluster 2024-03-07T03:47:27.306183+0000 mon.a (mon.0) 1850 : cluster 0 osdmap e1015: 8 total, 8 up, 8 in 2024-03-07T03:47:28.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:28 smithi145 bash[82329]: cluster 2024-03-07T03:47:27.626450+0000 mgr.y (mgr.34107) 1068 : cluster 0 pgmap v1708: 105 pgs: 105 active+clean; 758 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 108 KiB/s rd, 33 MiB/s wr, 160 op/s 2024-03-07T03:47:29.357 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 1 diff [7302669~1437061,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,226998752~6514249,264992561~1606876,279969482~1941282] 2024-03-07T03:47:29.357 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [7302669~1437061,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,226998752~6514249,264992561~1606876,279969482~1941282] 2024-03-07T03:47:29.392 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~4545426,155189248~8169298,167772160~15499621,188743680~10512397,226492416~7020585,264992561~1606876,279969482~1941282] 2024-03-07T03:47:29.393 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 2 diff [2318850~3094327,7302669~1437061,97961740~460363,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,214732945~2402879,226998752~6514249,247013430~638075,257695447~6054536,264661926~6914259,279969482~1941282,322156674~5263615,338329190~3592500,379305389~6030299] 2024-03-07T03:47:29.393 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~3094327,7302669~1437061,97961740~460363,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,214732945~2402879,226998752~6514249,247013430~638075,257695447~6054536,264661926~6914259,279969482~1941282,322156674~5263615,338329190~3592500,379305389~6030299] 2024-03-07T03:47:29.410 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~8739730,96468992~1953111,155189248~8169298,167772160~15499621,188743680~10512397,213909504~3226320,226492416~7020585,247013430~638075,255852544~7897439,264661926~6914259,279969482~1941282,318767104~8653185,335544320~6377370,377487360~7848328] 2024-03-07T03:47:29.410 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 3 diff [2318850~3094327,7302669~1437061,36454755~3492606,60790717~5974615,69531456~1516890,87852947~7066102,97961740~460363,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,214732945~2402879,226998752~6514249,247013430~638075,257695447~6054536,264661926~6914259,279969482~3936829,322156674~5263615,331086294~10835396,353776387~5206630,364517326~7139856,379305389~6030299] 2024-03-07T03:47:29.410 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~3094327,7302669~1437061,36454755~3492606,60790717~5974615,69531456~1516890,87852947~7066102,97961740~460363,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,214732945~2402879,226998752~6514249,247013430~638075,257695447~6054536,264661926~6914259,279969482~3936829,322156674~5263615,331086294~10835396,353776387~5206630,364517326~7139856,379305389~6030299] 2024-03-07T03:47:29.425 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~8739730,35908215~4039146,58720256~8045076,69531456~1516890,83886080~11032969,96468992~1953111,155189248~8169298,167772160~15499621,188743680~10512397,213909504~3226320,226492416~7020585,247013430~638075,255852544~7897439,264661926~6914259,279969482~3936829,318767104~23154586,352321536~6661481,360710144~10947038,377487360~7848328] 2024-03-07T03:47:29.425 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 4 diff [2318850~3094327,7302669~1437061,26015034~59803,36454755~3492606,60790717~5974615,69531456~1516890,87852947~7066102,97961740~460363,102460011~8177373,111781168~6525672,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,214732945~2402879,226998752~6514249,245529155~18220828,264661926~6914259,279969482~3936829,284525397~4534989,322156674~5263615,329627709~325719,331086294~15243300,353776387~5206630,364517326~7139856,379305389~6030299] 2024-03-07T03:47:29.425 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~3094327,7302669~1437061,26015034~59803,36454755~3492606,60790717~5974615,69531456~1516890,87852947~7066102,97961740~460363,102460011~8177373,111781168~6525672,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,214732945~2402879,226998752~6514249,245529155~18220828,264661926~6914259,279969482~3936829,284525397~4534989,322156674~5263615,329627709~325719,331086294~15243300,353776387~5206630,364517326~7139856,379305389~6030299] 2024-03-07T03:47:29.440 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~8739730,26015034~59803,35908215~4039146,58720256~8045076,69531456~1516890,83886080~11032969,96468992~1953111,100663296~9974088,111781168~6525672,155189248~8169298,167772160~15499621,188743680~10512397,213909504~3226320,226492416~7020585,245529155~18220828,264661926~6914259,279969482~9090904,318767104~27562490,352321536~6661481,360710144~10947038,377487360~7848328] 2024-03-07T03:47:29.440 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 5 diff [2318850~3698821,7302669~1437061,26015034~59803,36454755~3492606,43820204~645623,60790717~5974615,69531456~1516890,84515256~10403793,97961740~460363,102460011~8177373,111781168~6525672,128110794~4019853,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,214732945~2402879,226998752~6514249,245529155~18220828,264661926~6914259,279969482~3936829,284525397~4534989,322156674~5263615,329627709~325719,331086294~17095149,353776387~5206630,362225914~9431268,379305389~6030299] 2024-03-07T03:47:29.440 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~3698821,7302669~1437061,26015034~59803,36454755~3492606,43820204~645623,60790717~5974615,69531456~1516890,84515256~10403793,97961740~460363,102460011~8177373,111781168~6525672,128110794~4019853,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,214732945~2402879,226998752~6514249,245529155~18220828,264661926~6914259,279969482~3936829,284525397~4534989,322156674~5263615,329627709~325719,331086294~17095149,353776387~5206630,362225914~9431268,379305389~6030299] 2024-03-07T03:47:29.452 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~8739730,26015034~59803,35908215~4039146,41943040~2522787,58720256~8045076,69531456~1516890,83886080~11032969,96468992~1953111,100663296~9974088,111781168~6525672,125829120~6301527,155189248~8169298,167772160~15499621,188743680~10512397,213909504~3226320,226492416~7020585,245529155~18220828,264661926~6914259,279969482~9090904,318767104~29414339,352321536~6661481,360710144~10947038,377487360~7848328] 2024-03-07T03:47:29.452 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 6 diff [2318850~7397726,13425281~5376438,26015034~59803,31786956~8160405,43820204~645623,60790717~5974615,69531456~1516890,84515256~10403793,95397413~180359,97961740~460363,102460011~8177373,111781168~6525672,126185278~6228220,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,214732945~2402879,223103880~1116779,226998752~6514249,245529155~18220828,264661926~6914259,279969482~3936829,284525397~4534989,304833665~5535071,322156674~5263615,329627709~325719,331086294~17095149,348725436~10257581,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.453 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~7397726,13425281~5376438,26015034~59803,31786956~8160405,43820204~645623,60790717~5974615,69531456~1516890,84515256~10403793,95397413~180359,97961740~460363,102460011~8177373,111781168~6525672,126185278~6228220,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,214732945~2402879,223103880~1116779,226998752~6514249,245529155~18220828,264661926~6914259,279969482~3936829,284525397~4534989,304833665~5535071,322156674~5263615,329627709~325719,331086294~17095149,348725436~10257581,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.464 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~9716576,12582912~6218807,26015034~59803,31786956~8160405,41943040~2522787,58720256~8045076,69531456~1516890,83886080~11691692,96468992~1953111,100663296~9974088,111781168~6525672,125829120~6584378,155189248~8169298,167772160~15499621,188743680~10512397,213909504~3226320,222298112~1922547,226492416~7020585,245529155~18220828,264661926~6914259,279969482~9090904,301989888~8378848,318767104~40215913,360710144~10947038,377487360~7848328,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.464 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 7 diff [2318850~16482869,26015034~59803,31786956~9842342,43820204~645623,52568246~3756564,60790717~5974615,69531456~1516890,84515256~12403044,97961740~460363,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,214732945~2402879,223103880~1116779,226998752~6514249,243436846~20313137,264661926~6914259,279969482~3936829,284525397~4534989,304833665~5535071,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.465 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,26015034~59803,31786956~9842342,43820204~645623,52568246~3756564,60790717~5974615,69531456~1516890,84515256~12403044,97961740~460363,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,156381382~6977164,171217122~7466361,178926356~4345425,192552650~6703427,214732945~2402879,223103880~1116779,226998752~6514249,243436846~20313137,264661926~6914259,279969482~3936829,284525397~4534989,304833665~5535071,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.475 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,26015034~59803,31786956~9842342,41943040~2522787,50331648~5993162,58720256~8045076,69531456~1516890,83886080~14536023,100663296~9974088,111781168~10683643,125829120~6584378,134217728~7035130,146800640~1416657,155189248~8169298,167772160~15499621,188743680~10512397,213909504~3226320,222298112~1922547,226492416~7020585,243436846~20313137,264661926~6914259,279969482~9090904,301989888~8378848,318767104~52890078,377487360~7848328,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.475 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 8 diff [2318850~16482869,26015034~59803,31786956~11047141,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,69531456~1516890,71963918~4427418,84405198~12513102,97961740~460363,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,156381382~11790402,171217122~7466361,178926356~4345425,190129935~302950,192552650~6703427,214732945~2761071,223103880~1116779,226998752~6514249,243436846~20684377,264661926~6914259,279969482~3936829,284525397~4534989,304833665~5535071,311853868~2589929,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.475 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,26015034~59803,31786956~11047141,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,69531456~1516890,71963918~4427418,84405198~12513102,97961740~460363,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,156381382~11790402,171217122~7466361,178926356~4345425,190129935~302950,192552650~6703427,214732945~2761071,223103880~1116779,226998752~6514249,243436846~20684377,264661926~6914259,279969482~3936829,284525397~4534989,304833665~5535071,311853868~2589929,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.484 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,26015034~59803,31786956~12678871,46137344~3462084,50331648~5993162,58720256~8045076,69531456~1516890,71963918~4427418,83886080~14536023,100663296~9974088,111781168~10683643,125829120~6584378,134217728~7035130,146800640~1416657,155189248~28082533,188743680~10512397,213909504~3584512,222298112~1922547,226492416~7020585,243436846~20684377,264661926~6914259,279969482~9090904,301989888~8378848,311853868~2589929,318767104~52890078,377487360~7848328,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.484 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 9 diff [2318850~16482869,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,67520406~8870930,84405198~12513102,97961740~460363,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,156381382~11790402,171217122~7466361,178926356~9707135,190129935~302950,192552650~6703427,214732945~2761071,223103880~1116779,226998752~6514249,243436846~20684377,264661926~6914259,279969482~3936829,284525397~4534989,304833665~5535071,311853868~2589929,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.485 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,67520406~8870930,84405198~12513102,97961740~460363,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,156381382~11790402,171217122~7466361,178926356~9707135,190129935~302950,192552650~6703427,214732945~2761071,223103880~1116779,226998752~6514249,243436846~20684377,264661926~6914259,279969482~3936829,284525397~4534989,304833665~5535071,311853868~2589929,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.493 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,26015034~59803,31786956~12678871,46137344~3462084,50331648~5993162,58720256~8045076,67520406~8870930,83886080~14536023,100663296~9974088,111781168~10683643,125829120~6584378,134217728~7035130,146800640~1416657,155189248~33444243,188743680~10512397,213909504~3584512,222298112~1922547,226492416~7020585,243436846~20684377,264661926~6914259,279969482~9090904,301989888~8378848,311853868~2589929,318767104~52890078,377487360~7848328,394264576~18628827] 2024-03-07T03:47:29.493 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 10 diff [2318850~16482869,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,156381382~11790402,171217122~7466361,178926356~9707135,190129935~302950,192552650~6703427,202024823~1392556,207097364~4743737,214732945~2761071,223103880~1116779,226998752~6514249,243436846~20684377,264661926~6914259,279969482~3936829,284525397~7006898,304833665~5535071,311853868~2589929,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.493 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,156381382~11790402,171217122~7466361,178926356~9707135,190129935~302950,192552650~6703427,202024823~1392556,207097364~4743737,214732945~2761071,223103880~1116779,226998752~6514249,243436846~20684377,264661926~6914259,279969482~3936829,284525397~7006898,304833665~5535071,311853868~2589929,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.501 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,26015034~59803,31786956~12678871,46137344~3462084,50331648~5993162,58720256~17671080,83886080~14536023,100663296~9974088,111781168~10764248,125829120~6584378,134217728~7035130,146800640~1416657,155189248~33444243,188743680~10512397,201326592~2090787,207097364~4743737,213909504~3584512,222298112~1922547,226492416~7020585,243436846~20684377,264661926~6914259,279969482~11562813,301989888~8378848,311853868~2589929,318767104~52890078,373293056~12042632,394264576~18628827] 2024-03-07T03:47:29.501 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 11 diff [2318850~16482869,26015034~59803,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,156128498~12043286,171217122~7466361,178926356~9707135,190129935~302950,192552650~6703427,202024823~1392556,206070966~7558505,214732945~2761071,223103880~1116779,226998752~6514249,243436846~20684377,264661926~6914259,279969482~3936829,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.501 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,26015034~59803,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,156128498~12043286,171217122~7466361,178926356~9707135,190129935~302950,192552650~6703427,202024823~1392556,206070966~7558505,214732945~2761071,223103880~1116779,226998752~6514249,243436846~20684377,264661926~6914259,279969482~3936829,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.509 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,26015034~59803,31786956~17812472,50331648~5993162,58720256~17671080,83886080~14536023,100663296~9974088,111781168~10764248,125829120~6584378,134217728~7035130,146800640~1416657,155189248~33444243,188743680~10512397,201326592~2090787,206070966~7558505,213909504~3584512,222298112~1922547,226492416~7020585,243436846~20684377,264661926~6914259,279969482~19704639,301989888~12722603,318767104~52890078,373293056~12042632,394264576~18628827] 2024-03-07T03:47:29.509 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 12 diff [2318850~16482869,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~20085405,126185278~6228220,134570734~6682124,147158925~7285607,156128498~12043286,171217122~7466361,178926356~9707135,190129935~302950,192552650~6703427,202024823~1392556,206070966~7558505,214732945~2761071,223103880~1116779,226998752~6514249,243436846~20684377,264661926~6914259,279969482~3936829,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,395654001~4615410,401472293~11421110] 2024-03-07T03:47:29.509 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~20085405,126185278~6228220,134570734~6682124,147158925~7285607,156128498~12043286,171217122~7466361,178926356~9707135,190129935~302950,192552650~6703427,202024823~1392556,206070966~7558505,214732945~2761071,223103880~1116779,226998752~6514249,243436846~20684377,264661926~6914259,279969482~3936829,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,395654001~4615410,401472293~11421110] 2024-03-07T03:47:29.516 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,31786956~17812472,50331648~5993162,58720256~17671080,83886080~14536023,100663296~21882120,125829120~6584378,134217728~7035130,146800640~7643892,155189248~33444243,188743680~10512397,201326592~2090787,206070966~7558505,213909504~3584512,222298112~1922547,226492416~7020585,243436846~20684377,264661926~6914259,279969482~19704639,301989888~12722603,318767104~52890078,373293056~12042632,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.516 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 13 diff [2318850~16482869,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6228220,134570734~6682124,147158925~7285607,156128498~12043286,171217122~7466361,178926356~9707135,190129935~302950,192552650~10864729,206070966~7558505,214732945~2761071,223103880~1116779,226998752~6514249,243436846~28139339,279969482~3936829,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.517 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6228220,134570734~6682124,147158925~7285607,156128498~12043286,171217122~7466361,178926356~9707135,190129935~302950,192552650~10864729,206070966~7558505,214732945~2761071,223103880~1116779,226998752~6514249,243436846~28139339,279969482~3936829,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.524 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,31786956~17812472,50331648~5993162,58720256~17671080,79691776~42853640,125829120~6584378,134217728~7035130,146800640~7643892,155189248~33444243,188743680~14673699,206070966~7558505,213909504~3584512,222298112~1922547,226492416~7020585,243436846~28139339,279969482~19704639,301989888~12722603,318767104~52890078,373293056~12042632,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.524 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 14 diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6228220,134570734~7674958,145598567~33084916,178926356~9707135,190129935~302950,192552650~10864729,206070966~7558505,214732945~2761071,223103880~1116779,226998752~6514249,243436846~30203343,279969482~3936829,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.524 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6228220,134570734~7674958,145598567~33084916,178926356~9707135,190129935~302950,192552650~10864729,206070966~7558505,214732945~2761071,223103880~1116779,226998752~6514249,243436846~30203343,279969482~3936829,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.531 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~2702724,31786956~17812472,50331648~5993162,58720256~17671080,79691776~42853640,125829120~6584378,134217728~8027964,142606336~46027155,188743680~14673699,206070966~7558505,213909504~3584512,222298112~1922547,226492416~7020585,243436846~30203343,279969482~19704639,301989888~12722603,318767104~52890078,373293056~12042632,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.531 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 15 diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52484686~6706282,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~33084916,178926356~9707135,190129935~302950,192552650~10864729,206070966~7558505,214732945~2761071,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.531 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52484686~6706282,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~33084916,178926356~9707135,190129935~302950,192552650~10864729,206070966~7558505,214732945~2761071,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.538 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~2702724,31786956~17812472,50331648~26059688,79691776~42853640,125829120~6692395,134217728~8027964,142606336~46027155,188743680~14673699,206070966~7558505,213909504~3584512,222298112~1922547,226492416~11905974,239075328~34564861,276596525~23077596,301989888~12722603,318767104~52890078,373293056~12042632,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.538 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 16 diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~43034924,190129935~302950,192552650~10864729,206070966~7558505,214732945~2761071,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.539 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~43034924,190129935~302950,192552650~10864729,206070966~7558505,214732945~2761071,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.546 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~2702724,31786956~18155565,50331648~26059688,79691776~42853640,125829120~6692395,134217728~8027964,142606336~46027155,188743680~14673699,206070966~7558505,213909504~3584512,222298112~1922547,226492416~11905974,239075328~34564861,276596525~23077596,301989888~12722603,318767104~52890078,373293056~12042632,385875968~27017435] 2024-03-07T03:47:29.546 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 17 diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,60790717~15600619,77508972~1810144,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214732945~7988617,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~30296262,362225914~9431268,375226132~10109556,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.546 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,60790717~15600619,77508972~1810144,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214732945~7988617,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~30296262,362225914~9431268,375226132~10109556,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.554 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~2702724,31786956~18155565,50331648~28987468,79691776~42853640,125829120~6692395,134217728~8027964,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~12722603,318767104~52890078,373293056~12042632,385875968~27017435] 2024-03-07T03:47:29.554 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 18 diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~15600619,77508972~1810144,82227694~15105520,97961740~25381762,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214732945~7988617,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~10109556,389185474~23707929] 2024-03-07T03:47:29.554 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~15600619,77508972~1810144,82227694~15105520,97961740~25381762,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214732945~7988617,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~10109556,389185474~23707929] 2024-03-07T03:47:29.561 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51607520,79691776~43651726,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~12042632,385875968~27017435] 2024-03-07T03:47:29.561 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 19 diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:29.561 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:29.568 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:29.568 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 2 diff [2318850~3094327,97961740~460363,214732945~2402879,247013430~638075,257695447~6054536,264661926~6914259,322156674~5263615,338329190~3592500,379305389~6030299] 2024-03-07T03:47:29.568 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~3094327,97961740~460363,214732945~2402879,247013430~638075,257695447~6054536,264661926~6914259,322156674~5263615,338329190~3592500,379305389~6030299] 2024-03-07T03:47:29.575 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~5413177,96468992~1953111,213909504~3226320,247013430~638075,255852544~7897439,264661926~6914259,318767104~8653185,335544320~6377370,377487360~7848328] 2024-03-07T03:47:29.576 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 3 diff [2318850~3094327,36454755~3492606,60790717~5974615,69531456~1516890,87852947~7066102,97961740~460363,214732945~2402879,247013430~638075,257695447~6054536,264661926~6914259,280033366~3872945,322156674~5263615,331086294~10835396,353776387~5206630,364517326~7139856,379305389~6030299] 2024-03-07T03:47:29.576 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~3094327,36454755~3492606,60790717~5974615,69531456~1516890,87852947~7066102,97961740~460363,214732945~2402879,247013430~638075,257695447~6054536,264661926~6914259,280033366~3872945,322156674~5263615,331086294~10835396,353776387~5206630,364517326~7139856,379305389~6030299] 2024-03-07T03:47:29.583 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~5413177,35908215~4039146,58720256~8045076,69531456~1516890,83886080~11032969,96468992~1953111,213909504~3226320,247013430~638075,255852544~7897439,264661926~6914259,280033366~3872945,318767104~23154586,352321536~6661481,360710144~10947038,377487360~7848328] 2024-03-07T03:47:29.583 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 4 diff [2318850~3094327,26015034~59803,36454755~3492606,60790717~5974615,69531456~1516890,87852947~7066102,97961740~460363,102460011~8177373,111781168~6525672,214732945~2402879,245529155~18220828,264661926~6914259,280033366~3872945,284525397~4534989,322156674~5263615,329627709~325719,331086294~15243300,353776387~5206630,364517326~7139856,379305389~6030299] 2024-03-07T03:47:29.583 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~3094327,26015034~59803,36454755~3492606,60790717~5974615,69531456~1516890,87852947~7066102,97961740~460363,102460011~8177373,111781168~6525672,214732945~2402879,245529155~18220828,264661926~6914259,280033366~3872945,284525397~4534989,322156674~5263615,329627709~325719,331086294~15243300,353776387~5206630,364517326~7139856,379305389~6030299] 2024-03-07T03:47:29.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:29 smithi189 bash[66969]: cluster 2024-03-07T03:47:28.330546+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e1016: 8 total, 8 up, 8 in 2024-03-07T03:47:29.591 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~5413177,26015034~59803,35908215~4039146,58720256~8045076,69531456~1516890,83886080~11032969,96468992~1953111,100663296~9974088,111781168~6525672,213909504~3226320,245529155~18220828,264661926~6914259,280033366~9027020,318767104~27562490,352321536~6661481,360710144~10947038,377487360~7848328] 2024-03-07T03:47:29.591 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 5 diff [2318850~3698821,26015034~59803,36454755~3492606,43820204~645623,60790717~5974615,69531456~1516890,84515256~10403793,97961740~460363,102460011~8177373,111781168~6525672,128110794~4019853,214732945~2402879,245529155~18220828,264661926~6914259,280033366~3872945,284525397~4534989,322156674~5263615,329627709~325719,331086294~17095149,353776387~5206630,362225914~9431268,379305389~6030299] 2024-03-07T03:47:29.591 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~3698821,26015034~59803,36454755~3492606,43820204~645623,60790717~5974615,69531456~1516890,84515256~10403793,97961740~460363,102460011~8177373,111781168~6525672,128110794~4019853,214732945~2402879,245529155~18220828,264661926~6914259,280033366~3872945,284525397~4534989,322156674~5263615,329627709~325719,331086294~17095149,353776387~5206630,362225914~9431268,379305389~6030299] 2024-03-07T03:47:29.598 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~6017671,26015034~59803,35908215~4039146,41943040~2522787,58720256~8045076,69531456~1516890,83886080~11032969,96468992~1953111,100663296~9974088,111781168~6525672,125829120~6301527,213909504~3226320,245529155~18220828,264661926~6914259,280033366~9027020,318767104~29414339,352321536~6661481,360710144~10947038,377487360~7848328] 2024-03-07T03:47:29.598 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 6 diff [2318850~7397726,13425281~5376438,26015034~59803,31786956~8160405,43820204~645623,60790717~5974615,69531456~1516890,84515256~10403793,95397413~180359,97961740~460363,102460011~8177373,111781168~6525672,126185278~6228220,214732945~2402879,223103880~1116779,245529155~18220828,264661926~6914259,280033366~3872945,284525397~4534989,304833665~5535071,322156674~5263615,329627709~325719,331086294~17095149,348725436~10257581,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.598 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~7397726,13425281~5376438,26015034~59803,31786956~8160405,43820204~645623,60790717~5974615,69531456~1516890,84515256~10403793,95397413~180359,97961740~460363,102460011~8177373,111781168~6525672,126185278~6228220,214732945~2402879,223103880~1116779,245529155~18220828,264661926~6914259,280033366~3872945,284525397~4534989,304833665~5535071,322156674~5263615,329627709~325719,331086294~17095149,348725436~10257581,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.605 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~9716576,12582912~6218807,26015034~59803,31786956~8160405,41943040~2522787,58720256~8045076,69531456~1516890,83886080~11691692,96468992~1953111,100663296~9974088,111781168~6525672,125829120~6584378,213909504~3226320,222298112~1922547,245529155~18220828,264661926~6914259,280033366~9027020,301989888~8378848,318767104~40215913,360710144~10947038,377487360~7848328,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.605 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 7 diff [2318850~16482869,26015034~59803,31786956~9842342,43820204~645623,52568246~3756564,60790717~5974615,69531456~1516890,84515256~12403044,97961740~460363,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,214732945~2402879,223103880~1116779,243436846~20313137,264661926~6914259,280033366~3872945,284525397~4534989,304833665~5535071,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.605 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,26015034~59803,31786956~9842342,43820204~645623,52568246~3756564,60790717~5974615,69531456~1516890,84515256~12403044,97961740~460363,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,214732945~2402879,223103880~1116779,243436846~20313137,264661926~6914259,280033366~3872945,284525397~4534989,304833665~5535071,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.613 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,26015034~59803,31786956~9842342,41943040~2522787,50331648~5993162,58720256~8045076,69531456~1516890,83886080~14536023,100663296~9974088,111781168~10683643,125829120~6584378,134217728~7035130,146800640~1416657,213909504~3226320,222298112~1922547,243436846~20313137,264661926~6914259,280033366~9027020,301989888~8378848,318767104~52890078,377487360~7848328,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.613 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 8 diff [2318850~16482869,26015034~59803,31786956~11047141,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,69531456~1516890,71963918~4427418,84405198~12513102,97961740~460363,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,190129935~302950,214732945~2761071,223103880~1116779,243436846~20684377,264661926~6914259,280033366~3872945,284525397~4534989,304833665~5535071,311853868~2589929,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.613 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,26015034~59803,31786956~11047141,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,69531456~1516890,71963918~4427418,84405198~12513102,97961740~460363,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,190129935~302950,214732945~2761071,223103880~1116779,243436846~20684377,264661926~6914259,280033366~3872945,284525397~4534989,304833665~5535071,311853868~2589929,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.620 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,26015034~59803,31786956~12678871,46137344~3462084,50331648~5993162,58720256~8045076,69531456~1516890,71963918~4427418,83886080~14536023,100663296~9974088,111781168~10683643,125829120~6584378,134217728~7035130,146800640~1416657,163070735~5101049,190129935~302950,213909504~3584512,222298112~1922547,243436846~20684377,264661926~6914259,280033366~9027020,301989888~8378848,311853868~2589929,318767104~52890078,377487360~7848328,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.620 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 9 diff [2318850~16482869,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,67520406~8870930,84405198~12513102,97961740~460363,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,214732945~2761071,223103880~1116779,227584913~2698758,243436846~20684377,264661926~6914259,280033366~3872945,284525397~4534989,304833665~5535071,311853868~2589929,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.621 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,67520406~8870930,84405198~12513102,97961740~460363,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,214732945~2761071,223103880~1116779,227584913~2698758,243436846~20684377,264661926~6914259,280033366~3872945,284525397~4534989,304833665~5535071,311853868~2589929,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,379305389~6030299,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.627 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,26015034~59803,31786956~12678871,46137344~3462084,50331648~5993162,58720256~8045076,67520406~8870930,83886080~14536023,100663296~9974088,111781168~10683643,125829120~6584378,134217728~7035130,146800640~1416657,163070735~5101049,180869607~7763884,190129935~302950,213909504~3584512,222298112~1922547,227584913~2698758,243436846~20684377,264661926~6914259,280033366~9027020,301989888~8378848,311853868~2589929,318767104~52890078,377487360~7848328,394264576~18628827] 2024-03-07T03:47:29.627 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 10 diff [2318850~16482869,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,214732945~2761071,223103880~1116779,227584913~2698758,243436846~20684377,264661926~6914259,280033366~3872945,284525397~7006898,304833665~5535071,311853868~2589929,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.627 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,214732945~2761071,223103880~1116779,227584913~2698758,243436846~20684377,264661926~6914259,280033366~3872945,284525397~7006898,304833665~5535071,311853868~2589929,322156674~5263615,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.634 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,26015034~59803,31786956~12678871,46137344~3462084,50331648~5993162,58720256~17671080,83886080~14536023,100663296~9974088,111781168~10764248,125829120~6584378,134217728~7035130,146800640~1416657,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,207097364~4743737,213909504~3584512,222298112~1922547,227584913~2698758,243436846~20684377,264661926~6914259,280033366~11498929,301989888~8378848,311853868~2589929,318767104~52890078,373293056~12042632,394264576~18628827] 2024-03-07T03:47:29.634 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 11 diff [2318850~16482869,26015034~59803,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,214732945~2761071,223103880~1116779,227584913~2698758,243436846~20684377,264661926~6914259,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.634 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,26015034~59803,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,214732945~2761071,223103880~1116779,227584913~2698758,243436846~20684377,264661926~6914259,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.641 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,26015034~59803,31786956~17812472,50331648~5993162,58720256~17671080,83886080~14536023,100663296~9974088,111781168~10764248,125829120~6584378,134217728~7035130,146800640~1416657,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,213909504~3584512,222298112~1922547,227584913~2698758,243436846~20684377,264661926~6914259,280033366~19640755,301989888~12722603,318767104~52890078,373293056~12042632,394264576~18628827] 2024-03-07T03:47:29.641 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 12 diff [2318850~16482869,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~20085405,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,214732945~2761071,223103880~1116779,227584913~2698758,243436846~20684377,264661926~6914259,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,395654001~4615410,401472293~11421110] 2024-03-07T03:47:29.641 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,97961740~460363,102460011~20085405,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,214732945~2761071,223103880~1116779,227584913~2698758,243436846~20684377,264661926~6914259,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,395654001~4615410,401472293~11421110] 2024-03-07T03:47:29.649 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,31786956~17812472,50331648~5993162,58720256~17671080,83886080~14536023,100663296~21882120,125829120~6584378,134217728~7035130,146800640~7643892,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,213909504~3584512,222298112~1922547,227584913~2698758,243436846~20684377,264661926~6914259,280033366~19640755,301989888~12722603,318767104~52890078,373293056~12042632,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.649 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 13 diff [2318850~16482869,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,214732945~2761071,223103880~1116779,227584913~2698758,243436846~28139339,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.649 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,214732945~2761071,223103880~1116779,227584913~2698758,243436846~28139339,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.655 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,31786956~17812472,50331648~5993162,58720256~17671080,79691776~42853640,125829120~6584378,134217728~7035130,146800640~7643892,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,213909504~3584512,222298112~1922547,227584913~2698758,243436846~28139339,280033366~19640755,301989888~12722603,318767104~52890078,373293056~12042632,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.655 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 14 diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6228220,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,214732945~2761071,223103880~1116779,227584913~2698758,243436846~30203343,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.655 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6228220,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,214732945~2761071,223103880~1116779,227584913~2698758,243436846~30203343,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.662 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~2702724,31786956~17812472,50331648~5993162,58720256~17671080,79691776~42853640,125829120~6584378,134217728~8027964,142606336~16943513,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,213909504~3584512,222298112~1922547,227584913~2698758,243436846~30203343,280033366~19640755,301989888~12722603,318767104~52890078,373293056~12042632,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.662 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 15 diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52484686~6706282,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,214732945~2761071,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.662 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52484686~6706282,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,214732945~2761071,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.670 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~2702724,31786956~17812472,50331648~26059688,79691776~42853640,125829120~6692395,134217728~8027964,142606336~16943513,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,213909504~3584512,222298112~1922547,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~12722603,318767104~52890078,373293056~12042632,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.670 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 16 diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,214732945~2761071,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.670 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,60790717~15600619,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,214732945~2761071,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~10109556,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.676 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~2702724,31786956~18155565,50331648~26059688,79691776~42853640,125829120~6692395,134217728~8027964,142606336~16943513,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,213909504~3584512,222298112~1922547,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~12722603,318767104~52890078,373293056~12042632,385875968~27017435] 2024-03-07T03:47:29.676 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 17 diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,60790717~15600619,77508972~1810144,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214732945~7988617,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~30296262,362225914~9431268,375226132~10109556,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.676 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,60790717~15600619,77508972~1810144,82227694~15105520,97961740~24583676,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214732945~7988617,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~9878826,320002472~7417817,329627709~325719,331086294~30296262,362225914~9431268,375226132~10109556,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.683 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~2702724,31786956~18155565,50331648~28987468,79691776~42853640,125829120~6692395,134217728~8027964,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~12722603,318767104~52890078,373293056~12042632,385875968~27017435] 2024-03-07T03:47:29.683 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 18 diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~15600619,77508972~1810144,82227694~15105520,97961740~25381762,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214732945~7988617,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~10109556,389185474~23707929] 2024-03-07T03:47:29.683 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~15600619,77508972~1810144,82227694~15105520,97961740~25381762,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214732945~7988617,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~10109556,389185474~23707929] 2024-03-07T03:47:29.690 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51607520,79691776~43651726,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~12042632,385875968~27017435] 2024-03-07T03:47:29.690 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 19 diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:29.691 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:29.698 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:29.698 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 3 diff [36454755~3492606,60790717~5974615,69531456~1516890,87852947~7066102,280033366~3872945,331086294~8284072,353776387~5206630,364517326~7139856] 2024-03-07T03:47:29.698 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [36454755~3492606,60790717~5974615,69531456~1516890,87852947~7066102,280033366~3872945,331086294~8284072,353776387~5206630,364517326~7139856] 2024-03-07T03:47:29.705 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [35908215~4039146,58720256~8045076,69531456~1516890,83886080~11032969,280033366~3872945,327420289~11950077,352321536~6661481,360710144~10947038] 2024-03-07T03:47:29.705 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 4 diff [26015034~59803,36454755~3492606,60790717~5974615,69531456~1516890,87852947~7066102,102460011~8177373,111781168~6525672,245529155~12485317,280033366~3872945,284525397~4534989,329627709~325719,331086294~9566430,341629853~4699741,353776387~5206630,364517326~7139856] 2024-03-07T03:47:29.705 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [26015034~59803,36454755~3492606,60790717~5974615,69531456~1516890,87852947~7066102,102460011~8177373,111781168~6525672,245529155~12485317,280033366~3872945,284525397~4534989,329627709~325719,331086294~9566430,341629853~4699741,353776387~5206630,364517326~7139856] 2024-03-07T03:47:29.711 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [26015034~59803,35908215~4039146,58720256~8045076,69531456~1516890,83886080~11032969,100663296~9974088,111781168~6525672,245529155~12485317,280033366~9027020,327420289~13232435,341629853~4699741,352321536~6661481,360710144~10947038] 2024-03-07T03:47:29.711 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 5 diff [4366544~1651127,26015034~59803,36454755~3492606,43820204~645623,60790717~5974615,69531456~1516890,84515256~10403793,102460011~8177373,111781168~6525672,128110794~4019853,245529155~12485317,280033366~3872945,284525397~4534989,329627709~325719,331086294~9912052,341629853~6551590,353776387~5206630,362225914~9431268] 2024-03-07T03:47:29.711 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4366544~1651127,26015034~59803,36454755~3492606,43820204~645623,60790717~5974615,69531456~1516890,84515256~10403793,102460011~8177373,111781168~6525672,128110794~4019853,245529155~12485317,280033366~3872945,284525397~4534989,329627709~325719,331086294~9912052,341629853~6551590,353776387~5206630,362225914~9431268] 2024-03-07T03:47:29.717 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4366544~1651127,26015034~59803,35908215~4039146,41943040~2522787,58720256~8045076,69531456~1516890,83886080~11032969,100663296~9974088,111781168~6525672,125829120~6301527,245529155~12485317,280033366~9027020,327420289~13578057,341629853~6551590,352321536~6661481,360710144~10947038] 2024-03-07T03:47:29.717 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 6 diff [4327490~5389086,13425281~5376438,26015034~59803,31786956~8160405,43820204~645623,60790717~5974615,69531456~1516890,84515256~10403793,95397413~180359,102460011~8177373,111781168~6525672,126185278~6228220,223103880~1116779,245529155~12485317,280033366~3872945,284525397~4534989,304833665~5535071,329627709~325719,331086294~9912052,341629853~6551590,348725436~10257581,362225914~9431268,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.718 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~5389086,13425281~5376438,26015034~59803,31786956~8160405,43820204~645623,60790717~5974615,69531456~1516890,84515256~10403793,95397413~180359,102460011~8177373,111781168~6525672,126185278~6228220,223103880~1116779,245529155~12485317,280033366~3872945,284525397~4534989,304833665~5535071,329627709~325719,331086294~9912052,341629853~6551590,348725436~10257581,362225914~9431268,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.724 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~5389086,12582912~6218807,26015034~59803,31786956~8160405,41943040~2522787,58720256~8045076,69531456~1516890,83886080~11691692,100663296~9974088,111781168~6525672,125829120~6584378,222298112~1922547,245529155~12485317,280033366~9027020,301989888~8378848,327420289~13578057,341629853~17353164,360710144~10947038,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.725 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 7 diff [4327490~14474229,26015034~59803,31786956~9842342,43820204~645623,52568246~3756564,60790717~5974615,69531456~1516890,84515256~12403044,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,223103880~1116779,243436846~14577626,280033366~3872945,284525397~4534989,304833665~5535071,329627709~325719,331086294~9912052,341629853~6551590,348725436~12657120,362225914~9431268,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.725 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,26015034~59803,31786956~9842342,43820204~645623,52568246~3756564,60790717~5974615,69531456~1516890,84515256~12403044,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,223103880~1116779,243436846~14577626,280033366~3872945,284525397~4534989,304833665~5535071,329627709~325719,331086294~9912052,341629853~6551590,348725436~12657120,362225914~9431268,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.732 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,26015034~59803,31786956~9842342,41943040~2522787,50331648~5993162,58720256~8045076,69531456~1516890,83886080~13032220,100663296~9974088,111781168~10683643,125829120~6584378,134217728~7035130,146800640~1416657,222298112~1922547,243436846~14577626,280033366~9027020,301989888~8378848,327420289~13578057,341629853~30027329,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.732 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 8 diff [4327490~14474229,26015034~59803,31786956~11047141,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,69531456~1516890,71963918~4427418,84405198~12513102,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,190129935~302950,216500076~993940,223103880~1116779,243436846~14577626,262307318~1813905,280033366~3872945,284525397~4534989,304833665~5535071,311853868~2589929,329627709~325719,331086294~9912052,341629853~6551590,348725436~12657120,362225914~9431268,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.732 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,26015034~59803,31786956~11047141,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,69531456~1516890,71963918~4427418,84405198~12513102,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,190129935~302950,216500076~993940,223103880~1116779,243436846~14577626,262307318~1813905,280033366~3872945,284525397~4534989,304833665~5535071,311853868~2589929,329627709~325719,331086294~9912052,341629853~6551590,348725436~12657120,362225914~9431268,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.739 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,26015034~59803,31786956~12678871,46137344~3462084,50331648~5993162,58720256~8045076,69531456~1516890,71963918~4427418,83886080~13032220,100663296~9974088,111781168~10683643,125829120~6584378,134217728~7035130,146800640~1416657,163070735~5101049,190129935~302950,216500076~993940,222298112~1922547,243436846~14577626,262307318~1813905,280033366~9027020,301989888~8378848,311853868~2589929,327420289~13578057,341629853~30027329,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.739 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 9 diff [4327490~14474229,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,67520406~8870930,84405198~12513102,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,280033366~3872945,284525397~4534989,304833665~5535071,311853868~2589929,329627709~325719,331086294~9912052,341629853~6551590,348725436~12657120,362225914~9431268,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.739 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,60790717~5974615,67520406~8870930,84405198~12513102,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,280033366~3872945,284525397~4534989,304833665~5535071,311853868~2589929,329627709~325719,331086294~9912052,341629853~6551590,348725436~12657120,362225914~9431268,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.746 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,26015034~59803,31786956~12678871,46137344~3462084,50331648~5993162,58720256~8045076,67520406~8870930,83886080~13032220,100663296~9974088,111781168~10683643,125829120~6584378,134217728~7035130,146800640~1416657,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,222298112~1922547,227584913~2698758,243436846~14577626,262307318~1813905,280033366~9027020,301989888~8378848,311853868~2589929,327420289~13578057,341629853~30027329,394264576~18628827] 2024-03-07T03:47:29.746 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 10 diff [4327490~14474229,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,280033366~3872945,284525397~7006898,304833665~5535071,311853868~2589929,329627709~325719,331086294~9912052,341629853~6551590,348725436~12657120,362225914~9431268,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.746 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,280033366~3872945,284525397~7006898,304833665~5535071,311853868~2589929,329627709~325719,331086294~9912052,341629853~6551590,348725436~12657120,362225914~9431268,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.753 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,26015034~59803,31786956~12678871,46137344~3462084,50331648~5993162,58720256~17671080,83886080~13032220,100663296~9974088,111781168~10764248,125829120~6584378,134217728~7035130,146800640~1416657,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,207097364~4743737,216500076~993940,222298112~1922547,227584913~2698758,243436846~14577626,262307318~1813905,280033366~11498929,301989888~8378848,311853868~2589929,327420289~13578057,341629853~30027329,373293056~7254534,394264576~18628827] 2024-03-07T03:47:29.753 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 11 diff [4327490~14474229,26015034~59803,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.753 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,26015034~59803,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.760 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,26015034~59803,31786956~17812472,50331648~5993162,58720256~17671080,83886080~13032220,100663296~9974088,111781168~10764248,125829120~6584378,134217728~7035130,146800640~1416657,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~14577626,262307318~1813905,280033366~19640755,301989888~12722603,320002472~4802377,327420289~44236893,373293056~7254534,394264576~18628827] 2024-03-07T03:47:29.760 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 12 diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,102460011~20085405,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~6296958,390408721~3258158,395654001~4615410,401472293~11421110] 2024-03-07T03:47:29.760 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,84405198~12513102,102460011~20085405,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~6296958,390408721~3258158,395654001~4615410,401472293~11421110] 2024-03-07T03:47:29.767 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,31786956~17812472,50331648~5993162,58720256~17671080,83886080~13032220,100663296~21882120,125829120~6584378,134217728~7035130,146800640~7643892,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~14577626,262307318~1813905,280033366~19640755,301989888~12722603,320002472~4802377,327420289~44236893,373293056~8230034,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.767 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 13 diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,98329964~24215452,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14614238,259345944~6967198,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.767 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,98329964~24215452,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14614238,259345944~6967198,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.774 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,31786956~17812472,50331648~5993162,58720256~17671080,79691776~17641438,98329964~24215452,125829120~6584378,134217728~7035130,146800640~7643892,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~14614238,259345944~6967198,280033366~19640755,301989888~12722603,320002472~4802377,327420289~44236893,373293056~8230034,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.774 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 14 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,98329964~24215452,126185278~6228220,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14614238,259345944~6967198,269734316~3905873,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.774 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52568246~3756564,60790717~15600619,82227694~15105520,98329964~24215452,126185278~6228220,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14614238,259345944~6967198,269734316~3905873,280033366~3872945,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.782 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~17812472,50331648~5993162,58720256~17671080,79691776~17641438,98329964~24215452,125829120~6584378,134217728~8027964,142606336~16943513,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~14614238,259345944~6967198,269734316~3905873,280033366~19640755,301989888~12722603,320002472~4802377,327420289~44236893,373293056~8230034,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.782 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 15 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52484686~6706282,60790717~15600619,82227694~15105520,98329964~24215452,126185278~6336237,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~9878826,320002472~4802377,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~8281362,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.782 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52484686~6706282,60790717~15600619,82227694~15105520,98329964~24215452,126185278~6336237,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~9878826,320002472~4802377,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~8281362,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.789 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~17812472,50331648~26059688,79691776~17641438,98329964~24215452,125829120~6692395,134217728~8027964,142606336~16943513,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~12722603,320002472~4802377,327420289~44236893,373293056~10214438,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.789 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 16 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,60790717~15600619,82227694~15105520,98329964~24215452,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~9878826,320002472~4802377,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.789 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,60790717~15600619,82227694~15105520,98329964~24215452,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~9878826,320002472~4802377,329627709~325719,331086294~17095149,348725436~12657120,362225914~9431268,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.796 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~18155565,50331648~26059688,79691776~17641438,98329964~24215452,125829120~6692395,134217728~8027964,142606336~16943513,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~12722603,320002472~4802377,327420289~44236893,373293056~10214438,385875968~27017435] 2024-03-07T03:47:29.796 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 17 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,60790717~15600619,77508972~1810144,82227694~15105520,98329964~24215452,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~9878826,320002472~4802377,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.796 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,60790717~15600619,77508972~1810144,82227694~15105520,98329964~24215452,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~9878826,320002472~4802377,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.803 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~18155565,50331648~28987468,79691776~17641438,98329964~24215452,125829120~6692395,134217728~8027964,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~8627485,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~12722603,320002472~4802377,327420289~44236893,373293056~10214438,385875968~27017435] 2024-03-07T03:47:29.803 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 18 diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~15600619,77508972~1810144,82227694~15105520,98329964~25013538,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,389185474~23707929] 2024-03-07T03:47:29.803 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~15600619,77508972~1810144,82227694~15105520,98329964~25013538,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,389185474~23707929] 2024-03-07T03:47:29.810 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51607520,79691776~17641438,98329964~25013538,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~8627485,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,385875968~27017435] 2024-03-07T03:47:29.811 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 19 diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:29.811 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:29.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:29 smithi145 bash[79878]: cluster 2024-03-07T03:47:28.330546+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e1016: 8 total, 8 up, 8 in 2024-03-07T03:47:29.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:29 smithi145 bash[82329]: cluster 2024-03-07T03:47:28.330546+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e1016: 8 total, 8 up, 8 in 2024-03-07T03:47:29.818 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:29.818 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 4 diff [26015034~59803,102460011~8177373,111781168~6525672,245529155~12485317,284525397~4534989,329627709~325719,338478793~2173931,341629853~4699741] 2024-03-07T03:47:29.818 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [26015034~59803,102460011~8177373,111781168~6525672,245529155~12485317,284525397~4534989,329627709~325719,338478793~2173931,341629853~4699741] 2024-03-07T03:47:29.824 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [26015034~59803,100663296~9974088,111781168~6525672,245529155~12485317,283906311~5154075,329627709~325719,338478793~2173931,341629853~4699741] 2024-03-07T03:47:29.825 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 5 diff [4366544~1651127,26015034~59803,43820204~645623,84515256~7087146,102460011~8177373,111781168~6525672,128110794~4019853,245529155~12485317,284525397~4534989,329627709~325719,334175736~6822610,341629853~6551590,362225914~7600999] 2024-03-07T03:47:29.825 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4366544~1651127,26015034~59803,43820204~645623,84515256~7087146,102460011~8177373,111781168~6525672,128110794~4019853,245529155~12485317,284525397~4534989,329627709~325719,334175736~6822610,341629853~6551590,362225914~7600999] 2024-03-07T03:47:29.832 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4366544~1651127,26015034~59803,41943040~2522787,84515256~7087146,100663296~9974088,111781168~6525672,125829120~6301527,245529155~12485317,283906311~5154075,329627709~325719,334175736~6822610,341629853~6551590,362225914~7600999] 2024-03-07T03:47:29.832 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 6 diff [4327490~5389086,13425281~5376438,26015034~59803,31786956~5603854,43820204~645623,84515256~7087146,95397413~180359,102460011~8177373,111781168~6525672,126185278~6228220,223103880~1116779,245529155~12485317,284525397~4534989,304833665~5535071,329627709~325719,334175736~6822610,341629853~6551590,348725436~6267586,362225914~7600999,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.832 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~5389086,13425281~5376438,26015034~59803,31786956~5603854,43820204~645623,84515256~7087146,95397413~180359,102460011~8177373,111781168~6525672,126185278~6228220,223103880~1116779,245529155~12485317,284525397~4534989,304833665~5535071,329627709~325719,334175736~6822610,341629853~6551590,348725436~6267586,362225914~7600999,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.839 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~5389086,12582912~6218807,26015034~59803,31786956~5603854,41943040~2522787,84515256~7087146,94919049~658723,100663296~9974088,111781168~6525672,125829120~6584378,222298112~1922547,245529155~12485317,283906311~5154075,301989888~8378848,329627709~325719,334175736~6822610,341629853~13363169,362225914~7600999,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.839 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 7 diff [4327490~14474229,26015034~59803,31786956~9842342,43820204~645623,52568246~3756564,84515256~7087146,94820503~2097797,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,223103880~1116779,243436846~14577626,284525397~4534989,304833665~5535071,329627709~325719,334175736~6822610,341629853~6551590,348725436~6267586,356430482~4952074,362225914~7600999,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.839 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,26015034~59803,31786956~9842342,43820204~645623,52568246~3756564,84515256~7087146,94820503~2097797,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,223103880~1116779,243436846~14577626,284525397~4534989,304833665~5535071,329627709~325719,334175736~6822610,341629853~6551590,348725436~6267586,356430482~4952074,362225914~7600999,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.846 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,26015034~59803,31786956~9842342,41943040~2522787,50331648~5993162,84515256~7087146,94820503~2097797,100663296~9974088,111781168~10683643,125829120~6584378,134217728~7035130,146800640~1416657,222298112~1922547,243436846~14577626,283906311~5154075,301989888~8378848,329627709~325719,334175736~6822610,341629853~13363169,356430482~4952074,362225914~7600999,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.846 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 8 diff [4327490~14474229,26015034~59803,31786956~11047141,43820204~645623,47609853~1989575,52568246~3756564,71963918~4427418,84405198~8165644,94820503~2097797,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,190129935~302950,216500076~993940,223103880~1116779,243436846~14577626,262307318~1813905,284525397~4534989,304833665~5535071,311853868~2589929,329627709~325719,334175736~6822610,341629853~6551590,348725436~6267586,356430482~4952074,362225914~7600999,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.846 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,26015034~59803,31786956~11047141,43820204~645623,47609853~1989575,52568246~3756564,71963918~4427418,84405198~8165644,94820503~2097797,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,190129935~302950,216500076~993940,223103880~1116779,243436846~14577626,262307318~1813905,284525397~4534989,304833665~5535071,311853868~2589929,329627709~325719,334175736~6822610,341629853~6551590,348725436~6267586,356430482~4952074,362225914~7600999,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.852 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,26015034~59803,31786956~12678871,46137344~3462084,50331648~5993162,71963918~4427418,84405198~8165644,94820503~2097797,100663296~9974088,111781168~10683643,125829120~6584378,134217728~7035130,146800640~1416657,163070735~5101049,190129935~302950,216500076~993940,222298112~1922547,243436846~14577626,262307318~1813905,283906311~5154075,301989888~8378848,311853868~2589929,329627709~325719,334175736~6822610,341629853~13363169,356430482~4952074,362225914~7600999,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.852 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 9 diff [4327490~14474229,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,67520406~8870930,84405198~8165644,94820503~2097797,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,284525397~4534989,304833665~5535071,311853868~2589929,329627709~325719,334175736~6822610,341629853~6551590,348725436~6267586,356226413~5156143,362225914~7600999,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.852 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,67520406~8870930,84405198~8165644,94820503~2097797,102460011~8177373,111781168~10683643,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,284525397~4534989,304833665~5535071,311853868~2589929,329627709~325719,334175736~6822610,341629853~6551590,348725436~6267586,356226413~5156143,362225914~7600999,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.859 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,26015034~59803,31786956~12678871,46137344~3462084,50331648~5993162,67520406~8870930,84405198~8165644,94820503~2097797,100663296~9974088,111781168~10683643,125829120~6584378,134217728~7035130,146800640~1416657,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,222298112~1922547,227584913~2698758,243436846~14577626,262307318~1813905,283906311~5154075,301989888~8378848,311853868~2589929,329627709~325719,334175736~6822610,341629853~13363169,356226413~5156143,362225914~7600999,394264576~18628827] 2024-03-07T03:47:29.859 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 10 diff [4327490~14474229,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,284525397~7006898,304833665~5535071,311853868~2589929,329627709~325719,334175736~6822610,341629853~6551590,348725436~6267586,356226413~5156143,362225914~7600999,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.859 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,26015034~59803,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,284525397~7006898,304833665~5535071,311853868~2589929,329627709~325719,334175736~6822610,341629853~6551590,348725436~6267586,356226413~5156143,362225914~7600999,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.867 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,26015034~59803,31786956~12678871,46137344~3462084,50331648~5993162,62513806~13877530,84405198~8165644,94820503~2097797,100663296~9974088,111781168~10764248,125829120~6584378,134217728~7035130,146800640~1416657,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,207097364~4743737,216500076~993940,222298112~1922547,227584913~2698758,243436846~14577626,262307318~1813905,283906311~7625984,301989888~8378848,311853868~2589929,329627709~325719,334175736~6822610,341629853~13363169,356226413~5156143,362225914~7600999,373293056~7254534,394264576~18628827] 2024-03-07T03:47:29.867 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 11 diff [4327490~14474229,26015034~59803,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.867 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,26015034~59803,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,102460011~8177373,111781168~10764248,126185278~6228220,135729567~5523291,147158925~1058372,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.874 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,26015034~59803,31786956~17812472,50331648~5993162,62513806~13877530,84405198~8165644,94820503~2097797,100663296~9974088,111781168~10764248,125829120~6584378,134217728~7035130,146800640~1416657,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~14577626,262307318~1813905,283906311~15767810,301989888~12722603,320002472~4802377,329627709~325719,334175736~20817286,356226413~5156143,362225914~7600999,373293056~7254534,394264576~18628827] 2024-03-07T03:47:29.874 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 12 diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,102460011~20085405,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~6296958,390408721~3258158,395654001~4615410,401472293~11421110] 2024-03-07T03:47:29.874 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,102460011~20085405,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14577626,262307318~1813905,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~6296958,390408721~3258158,395654001~4615410,401472293~11421110] 2024-03-07T03:47:29.881 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,31786956~17812472,50331648~5993162,62513806~13877530,84405198~8165644,94820503~2097797,100663296~21882120,125829120~6584378,134217728~7035130,146800640~7643892,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~14577626,262307318~1813905,283906311~15767810,301989888~12722603,320002472~4802377,329627709~325719,334175736~20817286,356226413~5156143,362225914~7600999,373293056~8230034,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.881 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 13 diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~24215452,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14614238,259345944~6967198,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.881 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~24215452,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14614238,259345944~6967198,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.888 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,31786956~17812472,50331648~5993162,62513806~13877530,79691776~12879066,94820503~2512711,98329964~24215452,125829120~6584378,134217728~7035130,146800640~7643892,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~14614238,259345944~6967198,283906311~15767810,301989888~12722603,320002472~4802377,329627709~325719,334175736~20817286,356226413~5156143,362225914~7600999,373293056~8230034,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.888 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 14 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~24215452,126185278~6228220,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14614238,259345944~6967198,269734316~3905873,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.888 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~24215452,126185278~6228220,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~14614238,259345944~6967198,269734316~3905873,284525397~7006898,292890177~6783944,304833665~9878826,320002472~4802377,329627709~325719,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.895 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~17812472,50331648~5993162,62513806~13877530,79691776~12879066,94820503~2512711,98329964~24215452,125829120~6584378,134217728~8027964,142606336~16943513,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~14614238,259345944~6967198,269734316~3905873,283906311~15767810,301989888~12722603,320002472~4802377,329627709~325719,334175736~20817286,356226413~5156143,362225914~7600999,373293056~8230034,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.895 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 15 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52484686~6706282,61973630~14417706,82227694~10343148,94820503~2512711,98329964~24215452,126185278~6336237,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~9878826,320002472~4802377,329627709~325719,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~8281362,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.895 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52484686~6706282,61973630~14417706,82227694~10343148,94820503~2512711,98329964~24215452,126185278~6336237,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~9878826,320002472~4802377,329627709~325719,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~8281362,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.903 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~17812472,50331648~8859320,61973630~14417706,79691776~12879066,94820503~2512711,98329964~24215452,125829120~6692395,134217728~8027964,142606336~16943513,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~12722603,320002472~4802377,329627709~325719,334175736~20817286,356226413~5156143,362225914~7600999,373293056~10214438,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.903 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 16 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,61973630~14417706,82227694~10343148,92768282~4564932,98329964~24215452,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~9878826,320002472~4802377,329627709~325719,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.903 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,61973630~14417706,82227694~10343148,92768282~4564932,98329964~24215452,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~9878826,320002472~4802377,329627709~325719,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.911 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~18155565,50331648~8859320,61973630~14417706,79691776~12879066,92768282~4564932,98329964~24215452,125829120~6692395,134217728~8027964,142606336~16943513,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~12722603,320002472~4802377,329627709~325719,334175736~20817286,356226413~5156143,362225914~7600999,373293056~10214438,385875968~27017435] 2024-03-07T03:47:29.911 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 17 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~24215452,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~9878826,320002472~4802377,329627709~325719,334175736~20817286,355622721~5759835,362225914~7600999,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.911 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~24215452,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~9878826,320002472~4802377,329627709~325719,334175736~20817286,355622721~5759835,362225914~7600999,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:29.918 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~18155565,50331648~8859320,61973630~17345486,79691776~12879066,92768282~4564932,98329964~24215452,125829120~6692395,134217728~8027964,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~8627485,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~12722603,320002472~4802377,329627709~325719,334175736~20817286,355622721~5759835,362225914~7600999,373293056~10214438,385875968~27017435] 2024-03-07T03:47:29.918 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 18 diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~25013538,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,389185474~23707929] 2024-03-07T03:47:29.918 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~25013538,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,389185474~23707929] 2024-03-07T03:47:29.925 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17345486,79691776~12879066,92768282~4564932,98329964~25013538,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~8627485,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,385875968~27017435] 2024-03-07T03:47:29.925 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 19 diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:29.925 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:29.932 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:29.932 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 5 diff [4366544~1651127,43820204~645623,84515256~7087146,128110794~4019853,334175736~6822610,342998383~5183060,362225914~7600999] 2024-03-07T03:47:29.932 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4366544~1651127,43820204~645623,84515256~7087146,128110794~4019853,334175736~6822610,342998383~5183060,362225914~7600999] 2024-03-07T03:47:29.939 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4366544~1651127,41943040~2522787,84515256~7087146,125829120~6301527,334175736~6822610,342998383~5183060,362225914~7600999] 2024-03-07T03:47:29.939 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 6 diff [4327490~5389086,13425281~5376438,31786956~5603854,43820204~645623,84515256~7087146,95397413~180359,126185278~6228220,223103880~1116779,304833665~5535071,334175736~6822610,342998383~5183060,348725436~6267586,362225914~7600999,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.939 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~5389086,13425281~5376438,31786956~5603854,43820204~645623,84515256~7087146,95397413~180359,126185278~6228220,223103880~1116779,304833665~5535071,334175736~6822610,342998383~5183060,348725436~6267586,362225914~7600999,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.946 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~5389086,12582912~6218807,31786956~5603854,41943040~2522787,84515256~7087146,94919049~658723,125829120~6584378,222298112~1922547,301989888~8378848,334175736~6822610,342998383~11994639,362225914~7600999,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.946 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 7 diff [4327490~14474229,31786956~9842342,43820204~645623,52568246~3756564,84515256~7087146,94820503~2097797,116976006~5488805,126185278~6228220,135729567~5523291,147158925~1058372,223103880~1116779,243436846~7263777,304833665~5535071,334175736~6822610,342998383~5183060,348725436~6267586,356430482~4952074,362225914~7600999,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.947 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,31786956~9842342,43820204~645623,52568246~3756564,84515256~7087146,94820503~2097797,116976006~5488805,126185278~6228220,135729567~5523291,147158925~1058372,223103880~1116779,243436846~7263777,304833665~5535071,334175736~6822610,342998383~5183060,348725436~6267586,356430482~4952074,362225914~7600999,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.954 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,31786956~9842342,41943040~2522787,50331648~5993162,84515256~7087146,94820503~2097797,116976006~5488805,125829120~6584378,134217728~7035130,146800640~1416657,222298112~1922547,243436846~7263777,301989888~8378848,334175736~6822610,342998383~11994639,356430482~4952074,362225914~7600999,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.954 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 8 diff [4327490~14474229,31786956~11047141,43820204~645623,47609853~1989575,52568246~3756564,71963918~4427418,84405198~8165644,94820503~2097797,116976006~5488805,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,190129935~302950,216500076~993940,223103880~1116779,243436846~7263777,256893024~361554,262307318~1813905,304833665~5535071,311853868~2589929,334175736~6822610,342998383~5183060,348725436~6267586,356430482~4952074,362225914~7600999,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.954 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,31786956~11047141,43820204~645623,47609853~1989575,52568246~3756564,71963918~4427418,84405198~8165644,94820503~2097797,116976006~5488805,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,190129935~302950,216500076~993940,223103880~1116779,243436846~7263777,256893024~361554,262307318~1813905,304833665~5535071,311853868~2589929,334175736~6822610,342998383~5183060,348725436~6267586,356430482~4952074,362225914~7600999,395654001~4615410,405400684~7492719] 2024-03-07T03:47:29.961 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,31786956~12678871,46137344~3462084,50331648~5993162,71963918~4427418,84405198~8165644,94820503~2097797,116976006~5488805,125829120~6584378,134217728~7035130,146800640~1416657,163070735~5101049,190129935~302950,216500076~993940,222298112~1922547,243436846~7263777,256893024~361554,262307318~1813905,301989888~8378848,311853868~2589929,334175736~6822610,342998383~11994639,356430482~4952074,362225914~7600999,394264576~6004835,402653184~10240219] 2024-03-07T03:47:29.961 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 9 diff [4327490~14474229,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,67520406~8870930,84405198~8165644,94820503~2097797,116976006~5488805,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,304833665~5535071,311853868~2589929,334175736~6822610,342998383~5183060,348725436~6267586,356226413~5156143,362225914~7600999,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.961 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,67520406~8870930,84405198~8165644,94820503~2097797,116976006~5488805,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,304833665~5535071,311853868~2589929,334175736~6822610,342998383~5183060,348725436~6267586,356226413~5156143,362225914~7600999,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.969 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,31786956~12678871,46137344~3462084,50331648~5993162,67520406~8870930,84405198~8165644,94820503~2097797,116976006~5488805,125829120~6584378,134217728~7035130,146800640~1416657,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,222298112~1922547,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,301989888~8378848,311853868~2589929,334175736~6822610,342998383~11994639,356226413~5156143,362225914~7600999,394264576~18628827] 2024-03-07T03:47:29.969 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 10 diff [4327490~14474229,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,304833665~5535071,311853868~2589929,334175736~6822610,342998383~5183060,348725436~6267586,356226413~5156143,362225914~7600999,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.969 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,31786956~11595862,43820204~645623,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,304833665~5535071,311853868~2589929,334175736~6822610,342998383~5183060,348725436~6267586,356226413~5156143,362225914~7600999,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.975 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,31786956~12678871,46137344~3462084,50331648~5993162,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,125829120~6584378,134217728~7035130,146800640~1416657,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,207097364~4743737,216500076~993940,222298112~1922547,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,301989888~8378848,311853868~2589929,334175736~6822610,342998383~11994639,356226413~5156143,362225914~7600999,373293056~7254534,394264576~18628827] 2024-03-07T03:47:29.976 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 11 diff [4327490~14474229,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,126185278~6228220,135729567~5523291,147158925~1058372,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.976 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,126185278~6228220,135729567~5523291,147158925~1058372,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:29.983 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,31786956~17812472,50331648~5993162,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,125829120~6584378,134217728~7035130,146800640~1416657,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~14193923,301989888~12722603,320002472~4802377,334175736~20817286,356226413~5156143,362225914~7600999,373293056~7254534,394264576~18628827] 2024-03-07T03:47:29.984 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 12 diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104516662~7567202,114533580~8011836,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~6296958,390408721~3258158,395654001~4615410,401472293~11421110] 2024-03-07T03:47:29.984 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104516662~7567202,114533580~8011836,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~6296958,390408721~3258158,395654001~4615410,401472293~11421110] 2024-03-07T03:47:29.991 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,31786956~17812472,50331648~5993162,62513806~13877530,84405198~8165644,94820503~2097797,104516662~7567202,114533580~8011836,125829120~6584378,134217728~7035130,146800640~7643892,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~14193923,301989888~12722603,320002472~4802377,334175736~20817286,356226413~5156143,362225914~7600999,373293056~8230034,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.991 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 13 diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.991 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.998 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,31786956~17812472,50331648~5993162,62513806~13877530,79691776~12879066,94820503~2512711,98329964~13753900,114533580~8011836,125829120~6584378,134217728~7035130,146800640~7643892,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,285480198~14193923,301989888~12722603,320002472~4802377,334175736~20817286,356226413~5156143,362225914~7600999,373293056~8230034,390070272~3596607,394264576~18628827] 2024-03-07T03:47:29.998 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 14 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6228220,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,269734316~3905873,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:29.998 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6228220,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,269734316~3905873,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:30.005 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~17812472,50331648~5993162,62513806~13877530,79691776~12879066,94820503~2512711,98329964~13753900,114533580~8011836,125829120~6584378,134217728~8027964,142606336~16943513,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,269734316~3905873,285480198~14193923,301989888~12722603,320002472~4802377,334175736~20817286,356226413~5156143,362225914~7600999,373293056~8230034,390070272~3596607,394264576~18628827] 2024-03-07T03:47:30.006 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 15 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52484686~6706282,61973630~14417706,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6336237,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~9878826,320002472~4802377,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~8281362,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:30.006 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52484686~6706282,61973630~14417706,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6336237,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~9878826,320002472~4802377,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~8281362,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:30.012 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~17812472,50331648~8859320,61973630~14417706,79691776~12879066,94820503~2512711,98329964~13753900,114533580~8011836,125829120~6692395,134217728~8027964,142606336~16943513,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,231680022~6718368,239075328~11625295,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~12722603,320002472~4802377,334175736~20817286,356226413~5156143,362225914~7600999,373293056~10214438,390070272~3596607,394264576~18628827] 2024-03-07T03:47:30.012 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 16 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,61973630~14417706,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~9878826,320002472~4802377,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:30.012 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,61973630~14417706,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~9878826,320002472~4802377,334175736~14005707,348725436~6267586,356226413~5156143,362225914~7600999,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:30.019 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~18155565,50331648~8859320,61973630~14417706,79691776~12879066,92768282~4564932,98329964~13753900,114533580~8011836,125829120~6692395,134217728~8027964,142606336~16943513,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,231680022~6718368,239075328~11625295,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~12722603,320002472~4802377,334175736~20817286,356226413~5156143,362225914~7600999,373293056~10214438,385875968~27017435] 2024-03-07T03:47:30.019 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 17 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~9878826,320002472~4802377,334175736~20817286,355622721~5759835,362225914~7600999,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:30.020 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~9878826,320002472~4802377,334175736~20817286,355622721~5759835,362225914~7600999,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:30.028 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~18155565,50331648~8859320,61973630~17345486,79691776~12879066,92768282~4564932,98329964~13753900,114533580~8011836,125829120~6692395,134217728~8027964,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~8627485,227584913~2698758,231680022~6718368,239075328~11625295,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~12722603,320002472~4802377,334175736~20817286,355622721~5759835,362225914~7600999,373293056~10214438,385875968~27017435] 2024-03-07T03:47:30.028 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 18 diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8809922,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,389185474~23707929] 2024-03-07T03:47:30.028 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8809922,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,389185474~23707929] 2024-03-07T03:47:30.035 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17345486,79691776~12879066,92768282~4564932,98329964~13753900,114533580~8809922,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~8627485,227584913~2698758,231680022~6718368,239075328~11625295,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,385875968~27017435] 2024-03-07T03:47:30.035 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 19 diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:30.035 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:30.040 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:30.040 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 6 diff [4327490~5389086,13425281~5376438,31786956~5603854,95397413~180359,126185278~6228220,223103880~1116779,304833665~5535071,348725436~6267586,395654001~4615410,405400684~7492719] 2024-03-07T03:47:30.040 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~5389086,13425281~5376438,31786956~5603854,95397413~180359,126185278~6228220,223103880~1116779,304833665~5535071,348725436~6267586,395654001~4615410,405400684~7492719] 2024-03-07T03:47:30.047 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~5389086,12582912~6218807,31786956~5603854,94919049~658723,126185278~6228220,222298112~1922547,301989888~8378848,348181443~6811579,394264576~6004835,402653184~10240219] 2024-03-07T03:47:30.047 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 7 diff [4327490~14474229,31786956~9842342,52568246~3756564,94820503~2097797,116976006~5488805,126185278~6228220,135729567~5523291,147158925~1058372,223103880~1116779,243436846~7263777,304833665~5535071,348725436~6267586,356430482~4952074,395654001~4615410,405400684~7492719] 2024-03-07T03:47:30.047 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,31786956~9842342,52568246~3756564,94820503~2097797,116976006~5488805,126185278~6228220,135729567~5523291,147158925~1058372,223103880~1116779,243436846~7263777,304833665~5535071,348725436~6267586,356430482~4952074,395654001~4615410,405400684~7492719] 2024-03-07T03:47:30.054 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,31786956~9842342,50331648~5993162,94820503~2097797,116976006~5488805,126185278~6228220,134217728~7035130,146800640~1416657,222298112~1922547,243436846~7263777,301989888~8378848,348181443~6811579,356430482~4952074,394264576~6004835,402653184~10240219] 2024-03-07T03:47:30.054 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 8 diff [4327490~14474229,31786956~11047141,47609853~1989575,52568246~3756564,71963918~4427418,84405198~8165644,94820503~2097797,116976006~5488805,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,190129935~302950,216500076~993940,223103880~1116779,243436846~7263777,256893024~361554,262307318~1813905,304833665~5535071,311853868~2589929,348725436~6267586,356430482~4952074,395654001~4615410,405400684~7492719] 2024-03-07T03:47:30.054 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,31786956~11047141,47609853~1989575,52568246~3756564,71963918~4427418,84405198~8165644,94820503~2097797,116976006~5488805,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,190129935~302950,216500076~993940,223103880~1116779,243436846~7263777,256893024~361554,262307318~1813905,304833665~5535071,311853868~2589929,348725436~6267586,356430482~4952074,395654001~4615410,405400684~7492719] 2024-03-07T03:47:30.060 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,31786956~11047141,46137344~3462084,50331648~5993162,71963918~4427418,84405198~8165644,94820503~2097797,116976006~5488805,126185278~6228220,134217728~7035130,146800640~1416657,163070735~5101049,190129935~302950,216500076~993940,222298112~1922547,243436846~7263777,256893024~361554,262307318~1813905,301989888~8378848,311853868~2589929,348181443~6811579,356430482~4952074,394264576~6004835,402653184~10240219] 2024-03-07T03:47:30.060 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 9 diff [4327490~14474229,31786956~11595862,47609853~1989575,52568246~3756564,67520406~8870930,84405198~8165644,94820503~2097797,116976006~5488805,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,304833665~5535071,311853868~2589929,348725436~6267586,356226413~5156143,395654001~4615410,402337569~10555834] 2024-03-07T03:47:30.060 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,31786956~11595862,47609853~1989575,52568246~3756564,67520406~8870930,84405198~8165644,94820503~2097797,116976006~5488805,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,304833665~5535071,311853868~2589929,348725436~6267586,356226413~5156143,395654001~4615410,402337569~10555834] 2024-03-07T03:47:30.067 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,31786956~11595862,46137344~3462084,50331648~5993162,67520406~8870930,84405198~8165644,94820503~2097797,116976006~5488805,126185278~6228220,134217728~7035130,146800640~1416657,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,222298112~1922547,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,301989888~8378848,311853868~2589929,348181443~6811579,356226413~5156143,394264576~18628827] 2024-03-07T03:47:30.068 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 10 diff [4327490~14474229,31786956~11595862,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,304833665~5535071,311853868~2589929,348725436~6267586,356226413~5156143,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:30.068 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,31786956~11595862,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,126185278~6228220,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,304833665~5535071,311853868~2589929,348725436~6267586,356226413~5156143,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:30.075 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,31786956~11595862,46137344~3462084,50331648~5993162,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,126185278~6228220,134217728~7035130,146800640~1416657,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,207097364~4743737,216500076~993940,222298112~1922547,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,301989888~8378848,311853868~2589929,348181443~6811579,356226413~5156143,373293056~7254534,394264576~18628827] 2024-03-07T03:47:30.075 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 11 diff [4327490~14474229,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,126185278~6228220,135729567~5523291,147158925~1058372,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,339547820~6235898,348725436~6267586,356226413~5156143,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:30.075 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,126185278~6228220,135729567~5523291,147158925~1058372,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,339547820~6235898,348725436~6267586,356226413~5156143,375226132~5321458,395654001~4615410,402337569~10555834] 2024-03-07T03:47:30.082 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,31786956~17812472,50331648~5993162,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,126185278~6228220,134217728~7035130,146800640~1416657,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~14193923,301989888~12722603,320002472~4802377,339547820~6235898,348181443~6811579,356226413~5156143,373293056~7254534,394264576~18628827] 2024-03-07T03:47:30.082 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 12 diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104516662~7567202,114533580~8011836,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,339547820~6235898,348725436~6267586,356226413~5156143,375226132~6296958,390408721~3258158,395654001~4615410,401472293~11421110] 2024-03-07T03:47:30.082 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104516662~7567202,114533580~8011836,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,339547820~6235898,348725436~6267586,356226413~5156143,375226132~6296958,390408721~3258158,395654001~4615410,401472293~11421110] 2024-03-07T03:47:30.089 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,31786956~17812472,50331648~5993162,62513806~13877530,84405198~8165644,94820503~2097797,104516662~7567202,114533580~8011836,126185278~6228220,134217728~7035130,146800640~7643892,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~14193923,301989888~12722603,320002472~4802377,339547820~6235898,348181443~6811579,356226413~5156143,373293056~8230034,390070272~3596607,394264576~18628827] 2024-03-07T03:47:30.089 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 13 diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,339547820~6235898,348725436~6267586,356226413~5156143,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:30.089 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6228220,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,339547820~6235898,348725436~6267586,356226413~5156143,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:30.096 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,31786956~17812472,50331648~5993162,62513806~13877530,79691776~12879066,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6228220,134217728~7035130,146800640~7643892,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,285480198~14193923,301989888~12722603,320002472~4802377,339547820~6235898,348181443~6811579,356226413~5156143,373293056~8230034,390070272~3596607,394264576~18628827] 2024-03-07T03:47:30.096 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 14 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6228220,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,269734316~3905873,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,339547820~6235898,348725436~6267586,356226413~5156143,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:30.096 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6228220,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,269734316~3905873,285480198~6052097,292890177~6783944,304833665~9878826,320002472~4802377,339547820~6235898,348725436~6267586,356226413~5156143,375226132~6296958,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:30.104 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~17812472,50331648~5993162,62513806~13877530,79691776~12879066,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6228220,134217728~8027964,142606336~16943513,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,269734316~3905873,285480198~14193923,301989888~12722603,320002472~4802377,339547820~6235898,348181443~6811579,356226413~5156143,373293056~8230034,390070272~3596607,394264576~18628827] 2024-03-07T03:47:30.104 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 15 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52484686~6706282,61973630~14417706,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6336237,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~9878826,320002472~4802377,335656785~10126933,348725436~6267586,356226413~5156143,375226132~8281362,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:30.104 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~15158595,47609853~1989575,52484686~6706282,61973630~14417706,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6336237,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~9878826,320002472~4802377,335656785~10126933,348725436~6267586,356226413~5156143,375226132~8281362,390408721~3258158,394824008~6549491,401472293~11421110] 2024-03-07T03:47:30.110 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~17812472,50331648~8859320,61973630~14417706,79691776~12879066,94820503~2512711,98329964~13753900,114533580~8011836,126185278~6336237,134217728~8027964,142606336~16943513,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,231680022~6718368,239075328~11625295,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~12722603,320002472~4802377,335656785~10126933,348181443~6811579,356226413~5156143,373293056~10214438,390070272~3596607,394264576~18628827] 2024-03-07T03:47:30.110 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 16 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,61973630~14417706,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~9878826,320002472~4802377,335656785~10126933,348725436~6267586,356226413~5156143,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:30.110 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,61973630~14417706,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~9878826,320002472~4802377,335656785~10126933,348725436~6267586,356226413~5156143,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:30.117 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~18155565,50331648~8859320,61973630~14417706,79691776~12879066,92768282~4564932,98329964~13753900,114533580~8011836,126185278~6336237,134217728~8027964,142606336~16943513,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,222298112~1922547,227584913~2698758,231680022~6718368,239075328~11625295,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~12722603,320002472~4802377,335656785~10126933,348181443~6811579,356226413~5156143,373293056~10214438,385875968~27017435] 2024-03-07T03:47:30.117 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 17 diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~9878826,320002472~4802377,335656785~10126933,346163747~8829275,355622721~5759835,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:30.118 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~2702724,31786956~18155565,52484686~6706282,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,126185278~6336237,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~9878826,320002472~4802377,335656785~10126933,346163747~8829275,355622721~5759835,375226132~8281362,389185474~12188025,401472293~11421110] 2024-03-07T03:47:30.125 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~2702724,31786956~18155565,50331648~8859320,61973630~17345486,79691776~12879066,92768282~4564932,98329964~13753900,114533580~8011836,126185278~6336237,134217728~8027964,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~8627485,227584913~2698758,231680022~6718368,239075328~11625295,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~12722603,320002472~4802377,335656785~10126933,346163747~8829275,355622721~5759835,373293056~10214438,385875968~27017435] 2024-03-07T03:47:30.125 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 18 diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8809922,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,389185474~23707929] 2024-03-07T03:47:30.125 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8809922,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,389185474~23707929] 2024-03-07T03:47:30.132 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17345486,79691776~12879066,92768282~4564932,98329964~13753900,114533580~8809922,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~8627485,227584913~2698758,231680022~6718368,239075328~11625295,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,385875968~27017435] 2024-03-07T03:47:30.132 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 19 diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:30.132 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:30.139 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:30.139 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 7 diff [8700349~6073181,36920789~4708509,52568246~3756564,94820503~2097797,116976006~5488805,135729567~5523291,147158925~1058372,243436846~7263777,356430482~4952074] 2024-03-07T03:47:30.139 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6073181,36920789~4708509,52568246~3756564,94820503~2097797,116976006~5488805,135729567~5523291,147158925~1058372,243436846~7263777,356430482~4952074] 2024-03-07T03:47:30.146 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6073181,36920789~4708509,50331648~5993162,94820503~2097797,116976006~5488805,134217728~7035130,146800640~1416657,243436846~7263777,356430482~4952074] 2024-03-07T03:47:30.147 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 8 diff [8700349~6073181,36920789~5913308,47609853~1989575,52568246~3756564,71963918~4427418,84405198~8165644,94820503~2097797,116976006~5488805,135729567~5523291,147158925~1058372,163070735~5101049,190129935~302950,216500076~993940,243436846~7263777,256893024~361554,262307318~1813905,311853868~2589929,356430482~4952074] 2024-03-07T03:47:30.147 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6073181,36920789~5913308,47609853~1989575,52568246~3756564,71963918~4427418,84405198~8165644,94820503~2097797,116976006~5488805,135729567~5523291,147158925~1058372,163070735~5101049,190129935~302950,216500076~993940,243436846~7263777,256893024~361554,262307318~1813905,311853868~2589929,356430482~4952074] 2024-03-07T03:47:30.153 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6073181,36920789~5913308,46137344~3462084,50331648~5993162,71963918~4427418,84405198~8165644,94820503~2097797,116976006~5488805,134217728~7035130,146800640~1416657,163070735~5101049,190129935~302950,216500076~993940,243436846~7263777,256893024~361554,262307318~1813905,311853868~2589929,356430482~4952074] 2024-03-07T03:47:30.154 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 9 diff [8700349~6073181,32517841~10864977,47609853~1989575,52568246~3756564,67520406~8870930,84405198~8165644,94820503~2097797,116976006~5488805,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,311853868~2589929,356226413~5156143,402337569~8017381] 2024-03-07T03:47:30.154 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6073181,32517841~10864977,47609853~1989575,52568246~3756564,67520406~8870930,84405198~8165644,94820503~2097797,116976006~5488805,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,311853868~2589929,356226413~5156143,402337569~8017381] 2024-03-07T03:47:30.161 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6073181,32517841~10864977,46137344~3462084,50331648~5993162,67520406~8870930,84405198~8165644,94820503~2097797,116976006~5488805,134217728~7035130,146800640~1416657,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,311853868~2589929,356226413~5156143,400269411~10085539] 2024-03-07T03:47:30.161 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 10 diff [8700349~6536521,32517841~10864977,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,311853868~2589929,356226413~5156143,375226132~5321458,402337569~8017381] 2024-03-07T03:47:30.161 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,32517841~10864977,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,135729567~5523291,147158925~1058372,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,311853868~2589929,356226413~5156143,375226132~5321458,402337569~8017381] 2024-03-07T03:47:30.169 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,32517841~10864977,46137344~3462084,50331648~5993162,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,134217728~7035130,146800640~1416657,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,207097364~4743737,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,311853868~2589929,356226413~5156143,373293056~7254534,400269411~10085539] 2024-03-07T03:47:30.169 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 11 diff [8700349~6536521,32517841~14427710,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,135729567~5523291,147158925~1058372,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~5156143,375226132~5321458,402337569~8017381] 2024-03-07T03:47:30.169 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,32517841~14427710,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,135729567~5523291,147158925~1058372,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~5156143,375226132~5321458,402337569~8017381] 2024-03-07T03:47:30.176 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,32517841~17081587,50331648~5993162,62513806~13877530,84405198~8165644,94820503~2097797,104659775~4064028,114533580~8011836,134217728~7035130,146800640~1416657,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~5156143,373293056~7254534,400269411~10085539] 2024-03-07T03:47:30.176 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 12 diff [8700349~6536521,24990107~1581751,32517841~14427710,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104516662~7567202,114533580~8011836,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~5156143,375226132~6296958,390408721~3258158,401472293~8882657] 2024-03-07T03:47:30.176 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,32517841~14427710,47609853~1989575,52568246~3756564,62513806~13877530,84405198~8165644,94820503~2097797,104516662~7567202,114533580~8011836,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~5156143,375226132~6296958,390408721~3258158,401472293~8882657] 2024-03-07T03:47:30.183 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,32517841~17081587,50331648~5993162,62513806~13877530,84405198~8165644,94820503~2097797,104516662~7567202,114533580~8011836,134217728~7035130,146800640~7643892,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,256893024~361554,262307318~1813905,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~5156143,373293056~8230034,390070272~3596607,400269411~10085539] 2024-03-07T03:47:30.184 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 13 diff [8700349~6536521,24990107~1581751,32517841~14427710,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~5156143,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.184 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,32517841~14427710,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,134570734~6682124,147158925~7285607,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~5156143,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.190 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,32517841~17081587,50331648~5993162,62513806~13877530,79691776~12879066,94820503~2512711,98329964~13753900,114533580~8011836,134217728~7035130,146800640~7643892,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~5156143,373293056~8230034,390070272~3596607,394824008~15530942] 2024-03-07T03:47:30.190 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 14 diff [8700349~6536521,24990107~1581751,27711596~2702724,32517841~14427710,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,269734316~3905873,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~5156143,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.190 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~2702724,32517841~14427710,47609853~1989575,52568246~3756564,62513806~13877530,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,269734316~3905873,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~5156143,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.197 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~2702724,32517841~17081587,50331648~5993162,62513806~13877530,79691776~12879066,94820503~2512711,98329964~13753900,114533580~8011836,134217728~8027964,142606336~16943513,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,243436846~7263777,251991784~6059300,259345944~6967198,269734316~3905873,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~5156143,373293056~8230034,390070272~3596607,394824008~15530942] 2024-03-07T03:47:30.197 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 15 diff [8700349~6536521,24990107~1581751,27711596~2702724,32517841~14427710,47609853~1989575,52484686~6706282,61973630~14417706,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~5156143,375226132~8281362,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.197 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~2702724,32517841~14427710,47609853~1989575,52484686~6706282,61973630~14417706,82227694~10343148,94820503~2512711,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~13951282,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~5156143,375226132~8281362,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.204 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~2702724,32517841~17081587,50331648~8859320,61973630~14417706,79691776~12879066,94820503~2512711,98329964~13753900,114533580~8011836,132162650~358865,134217728~8027964,142606336~16943513,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~5156143,373293056~10214438,390070272~3596607,394824008~15530942] 2024-03-07T03:47:30.204 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 16 diff [8700349~6536521,24990107~1581751,27711596~2702724,32517841~17424680,52484686~6706282,61973630~14417706,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~5156143,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.204 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~2702724,32517841~17424680,52484686~6706282,61973630~14417706,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~5156143,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.211 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~2702724,32517841~17424680,50331648~8859320,61973630~14417706,79691776~12879066,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134217728~8027964,142606336~16943513,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~5156143,373293056~10214438,385875968~26210041] 2024-03-07T03:47:30.211 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 17 diff [8700349~6536521,24990107~1581751,27711596~2702724,32517841~17424680,52484686~6706282,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~5759835,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.211 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~2702724,32517841~17424680,52484686~6706282,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~5759835,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.217 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~2702724,32517841~17424680,50331648~8859320,61973630~17345486,79691776~12879066,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134217728~8027964,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~5759835,373293056~10214438,385875968~26210041] 2024-03-07T03:47:30.217 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 18 diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8809922,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,389185474~22900535] 2024-03-07T03:47:30.218 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8809922,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,389185474~22900535] 2024-03-07T03:47:30.224 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17345486,79691776~12879066,92768282~4564932,98329964~13753900,114533580~8809922,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,385875968~26210041] 2024-03-07T03:47:30.224 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 19 diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:30.224 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:30.231 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:30.231 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 8 diff [38508433~4325664,47609853~1989575,71963918~4427418,84405198~8165644,163070735~5101049,190129935~302950,216500076~993940,256893024~361554,262307318~1813905,311853868~2589929] 2024-03-07T03:47:30.231 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [38508433~4325664,47609853~1989575,71963918~4427418,84405198~8165644,163070735~5101049,190129935~302950,216500076~993940,256893024~361554,262307318~1813905,311853868~2589929] 2024-03-07T03:47:30.237 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [38508433~4325664,46137344~3462084,71963918~4427418,84405198~8165644,163070735~5101049,190129935~302950,216500076~993940,256893024~361554,262307318~1813905,311853868~2589929] 2024-03-07T03:47:30.237 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 9 diff [32517841~10864977,47609853~1989575,67520406~8870930,84405198~8165644,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,224056275~91427,227584913~2698758,256893024~361554,262307318~1813905,311853868~2589929,356226413~335101,402337569~8017381] 2024-03-07T03:47:30.238 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [32517841~10864977,47609853~1989575,67520406~8870930,84405198~8165644,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,224056275~91427,227584913~2698758,256893024~361554,262307318~1813905,311853868~2589929,356226413~335101,402337569~8017381] 2024-03-07T03:47:30.244 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [32517841~10864977,46137344~3462084,67520406~8870930,84405198~8165644,163070735~5101049,180869607~7763884,190129935~302950,216500076~993940,224056275~91427,227584913~2698758,256893024~361554,262307318~1813905,311853868~2589929,356226413~335101,400269411~10085539] 2024-03-07T03:47:30.244 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 10 diff [11633950~3602920,32517841~10864977,47609853~1989575,62513806~13877530,84405198~8165644,104659775~4064028,114533580~8011836,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,216500076~993940,224056275~91427,227584913~2698758,256893024~361554,262307318~1813905,285480198~6052097,311853868~2589929,356226413~335101,358406858~1989264,375226132~5321458,402337569~8017381] 2024-03-07T03:47:30.244 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,32517841~10864977,47609853~1989575,62513806~13877530,84405198~8165644,104659775~4064028,114533580~8011836,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,207097364~4743737,216500076~993940,224056275~91427,227584913~2698758,256893024~361554,262307318~1813905,285480198~6052097,311853868~2589929,356226413~335101,358406858~1989264,375226132~5321458,402337569~8017381] 2024-03-07T03:47:30.251 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,32517841~10864977,46137344~3462084,62513806~13877530,84405198~8165644,104659775~4064028,114533580~8011836,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,207097364~4743737,216500076~993940,224056275~91427,227584913~2698758,256893024~361554,262307318~1813905,285480198~6052097,311853868~2589929,356226413~335101,358406858~1989264,373293056~7254534,400269411~10085539] 2024-03-07T03:47:30.251 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 11 diff [11633950~3602920,32517841~14427710,47609853~1989575,62513806~13877530,84405198~8165644,104659775~4064028,114533580~8011836,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~5321458,402337569~8017381] 2024-03-07T03:47:30.251 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,32517841~14427710,47609853~1989575,62513806~13877530,84405198~8165644,104659775~4064028,114533580~8011836,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~5321458,402337569~8017381] 2024-03-07T03:47:30.258 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,32517841~17081587,62513806~13877530,84405198~8165644,104659775~4064028,114533580~8011836,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,256893024~361554,262307318~1813905,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,373293056~7254534,400269411~10085539] 2024-03-07T03:47:30.258 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 12 diff [11633950~3602920,24990107~1581751,32517841~14427710,47609853~1989575,62513806~13877530,84405198~8165644,104516662~7567202,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~6296958,390408721~3258158,401472293~8882657] 2024-03-07T03:47:30.258 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,32517841~14427710,47609853~1989575,62513806~13877530,84405198~8165644,104516662~7567202,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,202024823~1392556,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,256893024~361554,262307318~1813905,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~6296958,390408721~3258158,401472293~8882657] 2024-03-07T03:47:30.267 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,32517841~17081587,62513806~13877530,84405198~8165644,104516662~7567202,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,201326592~2090787,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,256893024~361554,262307318~1813905,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,373293056~8230034,390070272~3596607,400269411~10085539] 2024-03-07T03:47:30.267 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 13 diff [11633950~3602920,24990107~1581751,32517841~14427710,47609853~1989575,62513806~13877530,82227694~10343148,96152654~1180560,98329964~13753900,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,251991784~6059300,259345944~6967198,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.267 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,32517841~14427710,47609853~1989575,62513806~13877530,82227694~10343148,96152654~1180560,98329964~13753900,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,251991784~6059300,259345944~6967198,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.272 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,32517841~17081587,62513806~13877530,79691776~12879066,96152654~1180560,98329964~13753900,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,163070735~5101049,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,251991784~6059300,259345944~6967198,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,373293056~8230034,390070272~3596607,394824008~15530942] 2024-03-07T03:47:30.272 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 14 diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~14427710,47609853~1989575,62513806~13877530,82227694~10343148,96152654~1180560,98329964~13753900,114533580~8011836,134570734~7674958,145598567~2134643,147989099~11560750,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,251991784~6059300,259345944~6967198,269734316~3905873,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.272 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~14427710,47609853~1989575,62513806~13877530,82227694~10343148,96152654~1180560,98329964~13753900,114533580~8011836,134570734~7674958,145598567~2134643,147989099~11560750,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,251991784~6059300,259345944~6967198,269734316~3905873,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.278 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~2702724,32517841~17081587,62513806~13877530,79691776~12879066,96152654~1180560,98329964~13753900,114533580~8011836,134570734~7674958,142606336~5126874,147989099~11560750,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,251991784~6059300,259345944~6967198,269734316~3905873,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,373293056~8230034,390070272~3596607,394824008~15530942] 2024-03-07T03:47:30.279 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 15 diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~14427710,47609853~1989575,52484686~6706282,61973630~14417706,82227694~10343148,96152654~1180560,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~335101,358406858~1989264,375226132~8281362,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.279 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~14427710,47609853~1989575,52484686~6706282,61973630~14417706,82227694~10343148,96152654~1180560,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~335101,358406858~1989264,375226132~8281362,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.285 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~2702724,32517841~17081587,52484686~6706282,61973630~14417706,79691776~12879066,96152654~1180560,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163070735~13112942,180869607~7763884,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~335101,358406858~1989264,373293056~10214438,390070272~3596607,394824008~15530942] 2024-03-07T03:47:30.285 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 16 diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~17424680,52484686~6706282,61973630~14417706,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~335101,358406858~1989264,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.285 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~17424680,52484686~6706282,61973630~14417706,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~335101,358406858~1989264,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.292 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~2702724,32517841~17424680,52484686~6706282,61973630~14417706,79691776~12879066,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,198381649~5035730,206070966~7558505,216500076~993940,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~335101,358406858~1989264,373293056~10214438,385875968~26210041] 2024-03-07T03:47:30.292 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 17 diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~17424680,52484686~6706282,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.292 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~17424680,52484686~6706282,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.298 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~2702724,32517841~17424680,52484686~6706282,61973630~17345486,79691776~12879066,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,373293056~10214438,385875968~26210041] 2024-03-07T03:47:30.298 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 18 diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8809922,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,389185474~22900535] 2024-03-07T03:47:30.299 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~14417706,77508972~1810144,82227694~10343148,92768282~4564932,98329964~13753900,114533580~8809922,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,389185474~22900535] 2024-03-07T03:47:30.305 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17345486,79691776~12879066,92768282~4564932,98329964~13753900,114533580~8809922,129286528~12959164,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,385875968~26210041] 2024-03-07T03:47:30.305 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 19 diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:30.305 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:30.311 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:30.311 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 9 diff [32517841~6046506,42356129~1026689,67520406~8127436,180869607~7763884,224056275~91427,227584913~2698758,356226413~335101,402337569~8017381] 2024-03-07T03:47:30.312 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [32517841~6046506,42356129~1026689,67520406~8127436,180869607~7763884,224056275~91427,227584913~2698758,356226413~335101,402337569~8017381] 2024-03-07T03:47:30.318 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [32517841~6046506,42356129~1026689,67520406~8127436,180869607~7763884,224056275~91427,227584913~2698758,356226413~335101,400269411~10085539] 2024-03-07T03:47:30.318 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 10 diff [11633950~3602920,32517841~6046506,42356129~1026689,62513806~13134036,89585825~97099,104659775~4064028,114533580~8011836,180869607~7763884,202024823~1392556,207097364~4743737,224056275~91427,227584913~2698758,285480198~6052097,356226413~335101,358406858~1989264,375226132~5321458,402337569~8017381] 2024-03-07T03:47:30.318 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,32517841~6046506,42356129~1026689,62513806~13134036,89585825~97099,104659775~4064028,114533580~8011836,180869607~7763884,202024823~1392556,207097364~4743737,224056275~91427,227584913~2698758,285480198~6052097,356226413~335101,358406858~1989264,375226132~5321458,402337569~8017381] 2024-03-07T03:47:30.325 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,32517841~6046506,42356129~1026689,62513806~13134036,89585825~97099,104659775~4064028,114533580~8011836,180869607~7763884,201326592~2090787,207097364~4743737,224056275~91427,227584913~2698758,285480198~6052097,356226413~335101,358406858~1989264,373293056~7254534,400269411~10085539] 2024-03-07T03:47:30.325 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 11 diff [11633950~3602920,32517841~6046506,42356129~4589422,62513806~13134036,89585825~97099,104659775~4064028,114533580~8011836,156128498~1778066,180869607~7763884,202024823~1392556,206070966~7558505,224056275~91427,227584913~2698758,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~5321458,402337569~8017381] 2024-03-07T03:47:30.325 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,32517841~6046506,42356129~4589422,62513806~13134036,89585825~97099,104659775~4064028,114533580~8011836,156128498~1778066,180869607~7763884,202024823~1392556,206070966~7558505,224056275~91427,227584913~2698758,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~5321458,402337569~8017381] 2024-03-07T03:47:30.331 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,32517841~6046506,42356129~4589422,62513806~13134036,89585825~97099,104659775~4064028,114533580~8011836,156128498~1778066,180869607~7763884,201326592~2090787,206070966~7558505,224056275~91427,227584913~2698758,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,373293056~7254534,400269411~10085539] 2024-03-07T03:47:30.332 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 12 diff [11633950~3602920,24990107~1581751,32517841~6046506,41028236~1102842,42356129~4589422,62513806~13134036,89585825~97099,104516662~7567202,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,180869607~7763884,202024823~1392556,206070966~7558505,224056275~91427,227584913~2698758,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~6296958,390408721~3258158,401472293~8882657] 2024-03-07T03:47:30.332 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,32517841~6046506,41028236~1102842,42356129~4589422,62513806~13134036,89585825~97099,104516662~7567202,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,180869607~7763884,202024823~1392556,206070966~7558505,224056275~91427,227584913~2698758,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~6296958,390408721~3258158,401472293~8882657] 2024-03-07T03:47:30.339 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,32517841~6046506,41028236~1102842,42356129~4589422,62513806~13134036,89585825~97099,104516662~7567202,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,180869607~7763884,201326592~2090787,206070966~7558505,224056275~91427,227584913~2698758,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,373293056~8230034,390070272~3596607,400269411~10085539] 2024-03-07T03:47:30.339 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 13 diff [11633950~3602920,24990107~1581751,32517841~6046506,41028236~1102842,42356129~4589422,62513806~13134036,82227694~6384327,89585825~97099,96152654~1180560,98329964~13753900,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,180869607~7763884,198381649~5035730,206070966~7558505,224056275~91427,227584913~2698758,251991784~6059300,259345944~6967198,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.339 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,32517841~6046506,41028236~1102842,42356129~4589422,62513806~13134036,82227694~6384327,89585825~97099,96152654~1180560,98329964~13753900,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,180869607~7763884,198381649~5035730,206070966~7558505,224056275~91427,227584913~2698758,251991784~6059300,259345944~6967198,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.347 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,32517841~6046506,41028236~1102842,42356129~4589422,62513806~13134036,79691776~8920245,89585825~97099,96152654~1180560,98329964~13753900,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,180869607~7763884,198381649~5035730,206070966~7558505,224056275~91427,227584913~2698758,251991784~6059300,259345944~6967198,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,373293056~8230034,390070272~3596607,394824008~15530942] 2024-03-07T03:47:30.347 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 14 diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~6046506,41028236~1102842,42157137~4788414,62513806~13134036,82227694~7777542,96152654~1180560,98329964~13753900,114533580~8011836,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,180869607~7763884,198381649~5035730,206070966~7558505,224056275~91427,227584913~2698758,251991784~6059300,259345944~6967198,269734316~3905873,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.347 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~6046506,41028236~1102842,42157137~4788414,62513806~13134036,82227694~7777542,96152654~1180560,98329964~13753900,114533580~8011836,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,180869607~7763884,198381649~5035730,206070966~7558505,224056275~91427,227584913~2698758,251991784~6059300,259345944~6967198,269734316~3905873,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.353 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~2702724,32517841~6046506,41028236~1102842,42157137~4788414,62513806~13134036,79691776~10313460,96152654~1180560,98329964~13753900,114533580~8011836,134570734~7674958,142606336~5126874,147989099~11560750,163152484~13031193,180869607~7763884,198381649~5035730,206070966~7558505,224056275~91427,227584913~2698758,251991784~6059300,259345944~6967198,269734316~3905873,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,356226413~335101,358406858~1989264,373293056~8230034,390070272~3596607,394824008~15530942] 2024-03-07T03:47:30.353 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 15 diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~14427710,52484686~6706282,61973630~13674212,82227694~7777542,96152654~1180560,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,180869607~7763884,198381649~5035730,206070966~7558505,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~335101,358406858~1989264,375226132~8281362,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.353 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~14427710,52484686~6706282,61973630~13674212,82227694~7777542,96152654~1180560,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,180869607~7763884,198381649~5035730,206070966~7558505,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~335101,358406858~1989264,375226132~8281362,390408721~3258158,394824008~6549491,401472293~8882657] 2024-03-07T03:47:30.359 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~2702724,32517841~14427710,52484686~6706282,61973630~13674212,79691776~10313460,96152654~1180560,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163152484~13031193,180869607~7763884,198381649~5035730,206070966~7558505,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~335101,358406858~1989264,373293056~10214438,390070272~3596607,394824008~15530942] 2024-03-07T03:47:30.359 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 16 diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~17424680,52484686~6706282,61973630~13674212,82227694~7777542,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~25481007,198381649~5035730,206070966~7558505,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~335101,358406858~1989264,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.359 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~17424680,52484686~6706282,61973630~13674212,82227694~7777542,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~25481007,198381649~5035730,206070966~7558505,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~335101,358406858~1989264,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.366 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~2702724,32517841~17424680,52484686~6706282,61973630~13674212,79691776~10313460,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163152484~25481007,198381649~5035730,206070966~7558505,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,356226413~335101,358406858~1989264,373293056~10214438,385875968~26210041] 2024-03-07T03:47:30.366 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 17 diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~17424680,52484686~6706282,61973630~13674212,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.366 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~2702724,32517841~17424680,52484686~6706282,61973630~13674212,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.373 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~2702724,32517841~17424680,52484686~6706282,61973630~13674212,76391336~2927780,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,373293056~10214438,385875968~26210041] 2024-03-07T03:47:30.373 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 18 diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~13674212,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~8809922,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,389185474~22900535] 2024-03-07T03:47:30.373 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~13674212,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~8809922,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,389185474~22900535] 2024-03-07T03:47:30.380 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~13674212,76391336~2927780,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~8809922,129286528~12959164,142606336~5126874,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,385875968~26210041] 2024-03-07T03:47:30.381 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 19 diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:30.381 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:30.387 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:30.387 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 10 diff [11633950~3602920,62513806~6402641,89585825~97099,104659775~4064028,114533580~8011836,202024823~1392556,207097364~4743737,285480198~6052097,358406858~1989264,375226132~5321458] 2024-03-07T03:47:30.387 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,62513806~6402641,89585825~97099,104659775~4064028,114533580~8011836,202024823~1392556,207097364~4743737,285480198~6052097,358406858~1989264,375226132~5321458] 2024-03-07T03:47:30.393 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,62513806~6402641,89585825~97099,104659775~4064028,114533580~8011836,201326592~2090787,207097364~4743737,285480198~6052097,358406858~1989264,373293056~7254534] 2024-03-07T03:47:30.393 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 11 diff [11633950~3602920,42576538~4369013,62513806~6402641,89585825~97099,104659775~4064028,114533580~8011836,156128498~1778066,202024823~1392556,206070966~7558505,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,358406858~1989264,375226132~5321458,404359095~5485689] 2024-03-07T03:47:30.393 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,42576538~4369013,62513806~6402641,89585825~97099,104659775~4064028,114533580~8011836,156128498~1778066,202024823~1392556,206070966~7558505,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,358406858~1989264,375226132~5321458,404359095~5485689] 2024-03-07T03:47:30.400 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,42576538~4369013,62513806~6402641,89585825~97099,104659775~4064028,114533580~8011836,156128498~1778066,201326592~2090787,206070966~7558505,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,358406858~1989264,373293056~7254534,404359095~5485689] 2024-03-07T03:47:30.400 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 12 diff [11633950~3602920,24990107~1581751,41028236~1102842,42576538~4369013,62513806~12959430,89585825~97099,104516662~7567202,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,202024823~1392556,206070966~7558505,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,358406858~1989264,375226132~6296958,390408721~3258158,401472293~8372491] 2024-03-07T03:47:30.400 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,41028236~1102842,42576538~4369013,62513806~12959430,89585825~97099,104516662~7567202,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,202024823~1392556,206070966~7558505,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,358406858~1989264,375226132~6296958,390408721~3258158,401472293~8372491] 2024-03-07T03:47:30.406 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,41028236~1102842,42576538~4369013,62513806~12959430,89585825~97099,104516662~7567202,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,201326592~2090787,206070966~7558505,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,358406858~1989264,373293056~8230034,390070272~3596607,401472293~8372491] 2024-03-07T03:47:30.406 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 13 diff [11633950~3602920,24990107~1581751,41028236~1102842,42576538~4369013,62513806~12959430,82227694~6384327,89585825~97099,96152654~1180560,98329964~13753900,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,198381649~5035730,206070966~7558505,251991784~6059300,259345944~6967198,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,358406858~1989264,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.406 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,41028236~1102842,42576538~4369013,62513806~12959430,82227694~6384327,89585825~97099,96152654~1180560,98329964~13753900,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,198381649~5035730,206070966~7558505,251991784~6059300,259345944~6967198,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,358406858~1989264,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.413 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,41028236~1102842,42576538~4369013,62513806~12959430,79691776~8920245,89585825~97099,96152654~1180560,98329964~13753900,114533580~8011836,134570734~3387789,147989099~6455433,156128498~1778066,198381649~5035730,206070966~7558505,251991784~6059300,259345944~6967198,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,358406858~1989264,373293056~8230034,390070272~3596607,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.413 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 14 diff [11633950~3602920,24990107~1581751,27711596~2702724,41028236~1102842,42157137~4788414,62513806~12959430,82227694~7777542,96152654~1180560,98329964~13753900,114533580~8011836,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,198381649~5035730,206070966~7558505,251991784~6059300,259345944~6967198,269734316~3905873,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,358406858~1989264,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.413 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~2702724,41028236~1102842,42157137~4788414,62513806~12959430,82227694~7777542,96152654~1180560,98329964~13753900,114533580~8011836,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,198381649~5035730,206070966~7558505,251991784~6059300,259345944~6967198,269734316~3905873,285480198~6052097,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,358406858~1989264,375226132~6296958,390408721~3258158,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.420 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~2702724,41028236~1102842,42157137~4788414,62513806~12959430,79691776~10313460,96152654~1180560,98329964~13753900,114533580~8011836,134570734~7674958,142606336~5126874,147989099~11560750,163152484~13031193,198381649~5035730,206070966~7558505,251991784~6059300,259345944~6967198,269734316~3905873,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,358406858~1989264,373293056~8230034,390070272~3596607,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.420 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 15 diff [11633950~3602920,24990107~1581751,27711596~2702724,36221741~10723810,52484686~6706282,61973630~13499606,82227694~7777542,96152654~1180560,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,198381649~5035730,206070966~7558505,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,358406858~1989264,375226132~8281362,390408721~3258158,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.420 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~2702724,36221741~10723810,52484686~6706282,61973630~13499606,82227694~7777542,96152654~1180560,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,198381649~5035730,206070966~7558505,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,358406858~1989264,375226132~8281362,390408721~3258158,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.426 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~2702724,36221741~10723810,52484686~6706282,61973630~13499606,79691776~10313460,96152654~1180560,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163152484~13031193,198381649~5035730,206070966~7558505,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,358406858~1989264,373293056~10214438,390070272~3596607,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.426 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 16 diff [11633950~3602920,24990107~1581751,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,82227694~7777542,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~17746333,198381649~5035730,206070966~7558505,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,358406858~1989264,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.426 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,82227694~7777542,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~17746333,198381649~5035730,206070966~7558505,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,358406858~1989264,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.433 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,79691776~10313460,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163152484~17746333,198381649~5035730,206070966~7558505,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,358406858~1989264,373293056~10214438,385875968~15497531,401472293~10613716] 2024-03-07T03:47:30.433 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 17 diff [11633950~3602920,24990107~1581751,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~17746333,186173555~1552003,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.433 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~17746333,186173555~1552003,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,375226132~8281362,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.441 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,76391336~2927780,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~8011836,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163152484~17746333,186173555~1552003,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,373293056~10214438,385875968~15497531,401472293~10613716] 2024-03-07T03:47:30.441 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 18 diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~13499606,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~8809922,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,389185474~22900535] 2024-03-07T03:47:30.441 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~13499606,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~8809922,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,389185474~22900535] 2024-03-07T03:47:30.448 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~8068414,36221741~23827780,61973630~13499606,76391336~2927780,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~8809922,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,215593174~7128388,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,373293056~10214438,385875968~26210041] 2024-03-07T03:47:30.448 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 19 diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:30.448 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:30.454 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:30.454 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 11 diff [42576538~4369013,156128498~1778066,206070966~7558505,287416788~2430416,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,404359095~5485689] 2024-03-07T03:47:30.454 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [42576538~4369013,156128498~1778066,206070966~7558505,287416788~2430416,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,404359095~5485689] 2024-03-07T03:47:30.460 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [42576538~4369013,156128498~1778066,206070966~7558505,287416788~2430416,291532295~8141826,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,404359095~5485689] 2024-03-07T03:47:30.460 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 12 diff [24990107~1581751,41028236~1102842,42576538~4369013,68299100~7174136,104516662~7567202,114766453~7272045,134570734~3387789,147989099~6455433,156128498~1778066,206070966~7558505,287416788~2430416,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,378511893~3011197,390408721~3258158,401472293~8372491] 2024-03-07T03:47:30.460 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,41028236~1102842,42576538~4369013,68299100~7174136,104516662~7567202,114766453~7272045,134570734~3387789,147989099~6455433,156128498~1778066,206070966~7558505,287416788~2430416,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,378511893~3011197,390408721~3258158,401472293~8372491] 2024-03-07T03:47:30.466 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,41028236~1102842,42576538~4369013,68299100~7174136,104516662~7567202,114766453~7272045,134570734~3387789,147989099~6455433,156128498~1778066,206070966~7558505,287416788~2430416,291532295~8141826,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,378511893~3011197,390070272~3596607,401472293~8372491] 2024-03-07T03:47:30.466 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 13 diff [24990107~1581751,41028236~1102842,42576538~4369013,64158202~11315034,82227694~6384327,96152654~1180560,98329964~13753900,114766453~7272045,134570734~3387789,147989099~6455433,156128498~1778066,198381649~4662106,206070966~7558505,251991784~6059300,259345944~6967198,287416788~2430416,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,378511893~3011197,390408721~3258158,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.466 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,41028236~1102842,42576538~4369013,64158202~11315034,82227694~6384327,96152654~1180560,98329964~13753900,114766453~7272045,134570734~3387789,147989099~6455433,156128498~1778066,198381649~4662106,206070966~7558505,251991784~6059300,259345944~6967198,287416788~2430416,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,378511893~3011197,390408721~3258158,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.475 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,41028236~1102842,42576538~4369013,64158202~11315034,79691776~8920245,96152654~1180560,98329964~13753900,114766453~7272045,134570734~3387789,147989099~6455433,156128498~1778066,198381649~4662106,206070966~7558505,251991784~6059300,259345944~6967198,287416788~2430416,291532295~8141826,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,378511893~3011197,390070272~3596607,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.475 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 14 diff [24990107~1581751,27711596~2702724,41028236~1102842,42157137~4788414,64158202~11315034,82227694~7777542,96152654~1180560,98329964~13753900,114766453~7272045,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,198381649~4662106,206070966~7558505,251991784~6059300,259345944~6967198,269734316~3905873,287416788~2430416,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,378511893~3011197,390408721~3258158,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.475 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~2702724,41028236~1102842,42157137~4788414,64158202~11315034,82227694~7777542,96152654~1180560,98329964~13753900,114766453~7272045,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,198381649~4662106,206070966~7558505,251991784~6059300,259345944~6967198,269734316~3905873,287416788~2430416,292890177~6783944,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,378511893~3011197,390408721~3258158,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.480 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~2702724,41028236~1102842,42157137~4788414,64158202~11315034,79691776~10313460,96152654~1180560,98329964~13753900,114766453~7272045,134570734~7674958,142606336~5126874,147989099~11560750,163152484~13031193,198381649~4662106,206070966~7558505,251991784~6059300,259345944~6967198,269734316~3905873,287416788~2430416,291532295~8141826,306214394~1374272,309137334~5575157,320002472~4802377,339547820~6235898,378511893~3011197,390070272~3596607,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.480 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 15 diff [24990107~1581751,27711596~2702724,36221741~10723810,52484686~6706282,61973630~13499606,82227694~7777542,96152654~1180560,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,198381649~4662106,206070966~7558505,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,378511893~4995601,390408721~3258158,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.480 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~2702724,36221741~10723810,52484686~6706282,61973630~13499606,82227694~7777542,96152654~1180560,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,198381649~4662106,206070966~7558505,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,378511893~4995601,390408721~3258158,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.487 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~2702724,36221741~10723810,52484686~6706282,61973630~13499606,79691776~10313460,96152654~1180560,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163152484~13031193,198381649~4662106,206070966~7558505,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,378511893~4995601,390070272~3596607,394824008~6549491,401472293~8372491] 2024-03-07T03:47:30.487 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 16 diff [24990107~1581751,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,82227694~7777542,92768282~4564932,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~17746333,198381649~4662106,206070966~7558505,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,378511893~4995601,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.487 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,82227694~7777542,92768282~4564932,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~17746333,198381649~4662106,206070966~7558505,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,378511893~4995601,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.494 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,79691776~10313460,92768282~4564932,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163152484~17746333,198381649~4662106,206070966~7558505,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,378511893~4995601,385875968~15497531,401472293~10613716] 2024-03-07T03:47:30.495 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 17 diff [24990107~1581751,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~17746333,186173555~1552003,191170311~4467874,198381649~4662106,206070966~7558505,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~665560,378511893~4995601,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.495 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~17746333,186173555~1552003,191170311~4467874,198381649~4662106,206070966~7558505,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~665560,378511893~4995601,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.502 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,76391336~2927780,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163152484~17746333,186173555~1552003,191170311~4467874,198381649~4662106,206070966~7558505,215593174~7128388,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~5575157,320002472~4802377,335656785~10126933,346163747~4187506,355622721~665560,378511893~4995601,385875968~15497531,401472293~10613716] 2024-03-07T03:47:30.502 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 18 diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~13499606,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~8577049,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,389185474~22900535] 2024-03-07T03:47:30.502 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~13499606,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~8577049,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,389185474~22900535] 2024-03-07T03:47:30.508 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~13499606,76391336~2927780,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~8577049,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,215593174~7128388,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,385875968~26210041] 2024-03-07T03:47:30.508 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 19 diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:30.508 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:30.515 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:30.515 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 12 diff [24990107~1581751,41028236~1102842,68299100~7174136,104516662~7567202,114766453~7272045,134570734~3387789,147989099~6455433,378511893~3011197,390408721~3258158,401472293~6686420] 2024-03-07T03:47:30.515 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,41028236~1102842,68299100~7174136,104516662~7567202,114766453~7272045,134570734~3387789,147989099~6455433,378511893~3011197,390408721~3258158,401472293~6686420] 2024-03-07T03:47:30.522 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,41028236~1102842,68299100~7174136,104516662~7567202,114766453~7272045,134570734~3387789,147989099~6455433,378511893~3011197,390070272~3596607,401472293~6686420] 2024-03-07T03:47:30.522 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 13 diff [24990107~1581751,41028236~1102842,64158202~11315034,82227694~6384327,96152654~1180560,98329964~13753900,114766453~7272045,134570734~3387789,147989099~6455433,198381649~4662106,251991784~6059300,259345944~6967198,378511893~3011197,390408721~3258158,394824008~6549491,401472293~6686420] 2024-03-07T03:47:30.522 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,41028236~1102842,64158202~11315034,82227694~6384327,96152654~1180560,98329964~13753900,114766453~7272045,134570734~3387789,147989099~6455433,198381649~4662106,251991784~6059300,259345944~6967198,378511893~3011197,390408721~3258158,394824008~6549491,401472293~6686420] 2024-03-07T03:47:30.529 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,41028236~1102842,64158202~11315034,79691776~8920245,96152654~1180560,98329964~13753900,114766453~7272045,134570734~3387789,147989099~6455433,198381649~4662106,251991784~6059300,259345944~6967198,378511893~3011197,390070272~3596607,394824008~6549491,401472293~6686420] 2024-03-07T03:47:30.530 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 14 diff [24990107~1581751,27711596~2702724,41028236~1102842,42157137~1900013,64158202~11315034,82227694~7777542,96152654~1180560,98329964~13753900,114766453~7272045,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,198381649~4662106,251991784~6059300,259345944~6967198,269734316~3905873,378511893~3011197,390408721~3258158,394824008~6549491,401472293~6686420] 2024-03-07T03:47:30.530 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~2702724,41028236~1102842,42157137~1900013,64158202~11315034,82227694~7777542,96152654~1180560,98329964~13753900,114766453~7272045,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,198381649~4662106,251991784~6059300,259345944~6967198,269734316~3905873,378511893~3011197,390408721~3258158,394824008~6549491,401472293~6686420] 2024-03-07T03:47:30.535 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~2702724,41028236~1102842,42157137~1900013,64158202~11315034,79691776~10313460,96152654~1180560,98329964~13753900,114766453~7272045,134570734~7674958,142606336~5126874,147989099~11560750,163152484~13031193,198381649~4662106,251991784~6059300,259345944~6967198,269734316~3905873,378511893~3011197,390070272~3596607,394824008~6549491,401472293~6686420] 2024-03-07T03:47:30.535 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 15 diff [24990107~1581751,27711596~2702724,36221741~8159531,52484686~6706282,61973630~13499606,82227694~7777542,96152654~1180560,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,198381649~4662106,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,378511893~4995601,390408721~3258158,394824008~6549491,401472293~6686420] 2024-03-07T03:47:30.535 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~2702724,36221741~8159531,52484686~6706282,61973630~13499606,82227694~7777542,96152654~1180560,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~13031193,198381649~4662106,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,378511893~4995601,390408721~3258158,394824008~6549491,401472293~6686420] 2024-03-07T03:47:30.541 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~2702724,36221741~8159531,52484686~6706282,61973630~13499606,79691776~10313460,96152654~1180560,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163152484~13031193,198381649~4662106,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,378511893~4995601,390070272~3596607,394824008~6549491,401472293~6686420] 2024-03-07T03:47:30.541 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 16 diff [24990107~1581751,27711596~2702724,36221741~8159531,45945810~3996711,52484686~6706282,61973630~13499606,82227694~7777542,92768282~4564932,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~17746333,198381649~4662106,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,378511893~4995601,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.541 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~2702724,36221741~8159531,45945810~3996711,52484686~6706282,61973630~13499606,82227694~7777542,92768282~4564932,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~17746333,198381649~4662106,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,378511893~4995601,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.548 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~2702724,36221741~8159531,45945810~3996711,52484686~6706282,61973630~13499606,79691776~10313460,92768282~4564932,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163152484~17746333,198381649~4662106,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,378511893~4995601,385875968~15497531,401472293~10613716] 2024-03-07T03:47:30.548 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 17 diff [24990107~1581751,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~17746333,186173555~1552003,191170311~4467874,198381649~4662106,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,346163747~4187506,355622721~665560,378511893~4995601,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.548 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,145598567~2134643,147989099~11560750,163152484~17746333,186173555~1552003,191170311~4467874,198381649~4662106,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,346163747~4187506,355622721~665560,378511893~4995601,389185474~12188025,401472293~10613716] 2024-03-07T03:47:30.555 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~2702724,36221741~13720780,52484686~6706282,61973630~13499606,76391336~2927780,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~7272045,132162650~358865,134570734~7674958,142606336~5126874,147989099~11560750,163152484~17746333,186173555~1552003,191170311~4467874,198381649~4662106,215593174~7128388,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,346163747~4187506,355622721~665560,378511893~4995601,385875968~15497531,401472293~10613716] 2024-03-07T03:47:30.555 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 18 diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~13499606,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~8577049,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,389185474~22900535] 2024-03-07T03:47:30.555 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~13499606,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~8577049,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,389185474~22900535] 2024-03-07T03:47:30.562 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~13499606,76391336~2927780,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~8577049,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,215593174~7128388,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,385875968~26210041] 2024-03-07T03:47:30.562 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 19 diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:30.562 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:30.623 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:30.623 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 13 diff [64158202~4966057,82227694~6384327,96152654~1180560,98329964~7380989,117334964~4480713,198381649~4662106,251991784~6059300,259345944~6967198,394824008~6549491] 2024-03-07T03:47:30.623 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [64158202~4966057,82227694~6384327,96152654~1180560,98329964~7380989,117334964~4480713,198381649~4662106,251991784~6059300,259345944~6967198,394824008~6549491] 2024-03-07T03:47:30.637 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [64158202~4966057,79691776~8920245,96152654~1180560,98329964~7380989,117334964~4480713,198381649~4662106,251991784~6059300,259345944~6967198,394824008~6549491] 2024-03-07T03:47:30.637 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 14 diff [27711596~2702724,42157137~1900013,64158202~4966057,82227694~7777542,96152654~1180560,98329964~7380989,117334964~4480713,137121160~5124532,145598567~2134643,151434756~8115093,163152484~13031193,198381649~4662106,251991784~6059300,259345944~6967198,269734316~3905873,394824008~6549491] 2024-03-07T03:47:30.637 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~2702724,42157137~1900013,64158202~4966057,82227694~7777542,96152654~1180560,98329964~7380989,117334964~4480713,137121160~5124532,145598567~2134643,151434756~8115093,163152484~13031193,198381649~4662106,251991784~6059300,259345944~6967198,269734316~3905873,394824008~6549491] 2024-03-07T03:47:30.645 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~2702724,42157137~1900013,64158202~4966057,79691776~10313460,96152654~1180560,98329964~7380989,117334964~4480713,137121160~5124532,142606336~5126874,151434756~8115093,163152484~13031193,198381649~4662106,251991784~6059300,259345944~6967198,269734316~3905873,394824008~6549491] 2024-03-07T03:47:30.645 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 15 diff [27711596~2702724,36221741~8159531,52484686~6706282,61973630~7150629,82227694~7777542,96152654~1180560,98329964~7380989,117334964~4480713,132162650~358865,137121160~5124532,145598567~2134643,151434756~8115093,163152484~13031193,198381649~4662106,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248,394824008~6549491] 2024-03-07T03:47:30.645 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~2702724,36221741~8159531,52484686~6706282,61973630~7150629,82227694~7777542,96152654~1180560,98329964~7380989,117334964~4480713,132162650~358865,137121160~5124532,145598567~2134643,151434756~8115093,163152484~13031193,198381649~4662106,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248,394824008~6549491] 2024-03-07T03:47:30.652 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~2702724,36221741~8159531,52484686~6706282,61973630~7150629,79691776~10313460,96152654~1180560,98329964~7380989,117334964~4480713,132162650~358865,137121160~5124532,142606336~5126874,151434756~8115093,163152484~13031193,198381649~4662106,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248,394824008~6549491] 2024-03-07T03:47:30.652 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 16 diff [27711596~2702724,36221741~8159531,45945810~3996711,52484686~6706282,61973630~7150629,69970383~2374636,82227694~7777542,92768282~4564932,98329964~7380989,117334964~4480713,132162650~358865,137121160~5124532,145598567~2134643,151434756~8115093,163152484~17746333,198381649~4662106,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248,389185474~12188025,405442075~6643934] 2024-03-07T03:47:30.652 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~2702724,36221741~8159531,45945810~3996711,52484686~6706282,61973630~7150629,69970383~2374636,82227694~7777542,92768282~4564932,98329964~7380989,117334964~4480713,132162650~358865,137121160~5124532,145598567~2134643,151434756~8115093,163152484~17746333,198381649~4662106,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248,389185474~12188025,405442075~6643934] 2024-03-07T03:47:30.659 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~2702724,36221741~8159531,45945810~3996711,52484686~6706282,61973630~7150629,69970383~2374636,79691776~10313460,92768282~4564932,98329964~7380989,117334964~4480713,132162650~358865,137121160~5124532,142606336~5126874,151434756~8115093,163152484~17746333,198381649~4662106,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248,385875968~15497531,405442075~6643934] 2024-03-07T03:47:30.659 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 17 diff [27711596~2702724,36221741~13720780,52484686~6706282,61973630~7150629,69970383~2374636,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~4480713,132162650~358865,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,186173555~1552003,191170311~4467874,198381649~4662106,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,346163747~4187506,355622721~665560,379281246~4226248,389185474~12188025,405442075~6643934] 2024-03-07T03:47:30.659 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~2702724,36221741~13720780,52484686~6706282,61973630~7150629,69970383~2374636,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~4480713,132162650~358865,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,186173555~1552003,191170311~4467874,198381649~4662106,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,346163747~4187506,355622721~665560,379281246~4226248,389185474~12188025,405442075~6643934] 2024-03-07T03:47:30.666 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~2702724,36221741~13720780,52484686~6706282,61973630~7150629,69970383~2374636,76391336~2927780,79691776~10313460,91110716~555941,92768282~4564932,98329964~7380989,117334964~4480713,132162650~358865,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,186173555~1552003,191170311~4467874,198381649~4662106,215593174~7128388,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,346163747~4187506,355622721~665560,379281246~4226248,385875968~15497531,405442075~6643934] 2024-03-07T03:47:30.666 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 18 diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~6008538,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,389185474~14762739,405442075~6643934] 2024-03-07T03:47:30.666 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,77508972~1810144,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~6008538,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,215593174~7128388,231680022~6718368,239142063~7324550,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,389185474~14762739,405442075~6643934] 2024-03-07T03:47:30.672 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,76391336~2927780,79691776~10313460,91110716~555941,92768282~4564932,98329964~7380989,117334964~6008538,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,215593174~7128388,231680022~6718368,239075328~7391285,251991784~6059300,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,385875968~18072245,405442075~6643934] 2024-03-07T03:47:30.672 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 19 diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:30.673 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:30.679 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,79691776~10313460,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:30.679 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to 14 diff [27711596~2702724,42157137~1900013,84122472~5882764,137121160~5124532,145598567~2134643,151434756~8115093,163152484~13031193,269734316~3905873] 2024-03-07T03:47:30.680 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~2702724,42157137~1900013,84122472~5882764,137121160~5124532,145598567~2134643,151434756~8115093,163152484~13031193,269734316~3905873] 2024-03-07T03:47:30.684 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~2702724,42157137~1900013,84122472~5882764,137121160~5124532,142606336~5126874,151434756~8115093,163152484~13031193,269734316~3905873] 2024-03-07T03:47:30.684 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to 15 diff [27711596~2702724,36221741~8159531,52484686~6706282,61973630~5522256,84122472~5882764,132162650~358865,137121160~5124532,145598567~2134643,151434756~8115093,163152484~13031193,231680022~6718368,239142063~7324550,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248] 2024-03-07T03:47:30.684 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~2702724,36221741~8159531,52484686~6706282,61973630~5522256,84122472~5882764,132162650~358865,137121160~5124532,145598567~2134643,151434756~8115093,163152484~13031193,231680022~6718368,239142063~7324550,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248] 2024-03-07T03:47:30.690 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~2702724,36221741~8159531,52484686~6706282,61973630~5522256,84122472~5882764,132162650~358865,137121160~5124532,142606336~5126874,151434756~8115093,163152484~13031193,231680022~6718368,239075328~7391285,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248] 2024-03-07T03:47:30.690 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to 16 diff [27711596~2702724,36221741~8159531,45945810~3996711,52484686~6706282,61973630~5522256,69970383~2374636,84122472~5882764,92768282~4310745,102928466~1050357,132162650~358865,137121160~5124532,145598567~2134643,151434756~8115093,163152484~17746333,200983077~344966,231680022~6718368,239142063~7324550,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248,389185474~6128674,405442075~6643934] 2024-03-07T03:47:30.690 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~2702724,36221741~8159531,45945810~3996711,52484686~6706282,61973630~5522256,69970383~2374636,84122472~5882764,92768282~4310745,102928466~1050357,132162650~358865,137121160~5124532,145598567~2134643,151434756~8115093,163152484~17746333,200983077~344966,231680022~6718368,239142063~7324550,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248,389185474~6128674,405442075~6643934] 2024-03-07T03:47:30.696 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~2702724,36221741~8159531,45945810~3996711,52484686~6706282,61973630~5522256,69970383~2374636,84122472~5882764,92768282~4310745,102928466~1050357,132162650~358865,137121160~5124532,142606336~5126874,151434756~8115093,163152484~17746333,200983077~344966,231680022~6718368,239075328~7391285,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248,385875968~9438180,405442075~6643934] 2024-03-07T03:47:30.696 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to 17 diff [27711596~2702724,36221741~13720780,52484686~6706282,61973630~5750354,69970383~2374636,77508972~1810144,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,132162650~358865,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,186173555~1552003,191170311~4467874,200983077~344966,215593174~7128388,231680022~6718368,239142063~7324550,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,346163747~4187506,355622721~665560,379281246~4226248,389185474~6128674,405442075~6643934] 2024-03-07T03:47:30.696 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~2702724,36221741~13720780,52484686~6706282,61973630~5750354,69970383~2374636,77508972~1810144,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,132162650~358865,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,186173555~1552003,191170311~4467874,200983077~344966,215593174~7128388,231680022~6718368,239142063~7324550,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,346163747~4187506,355622721~665560,379281246~4226248,389185474~6128674,405442075~6643934] 2024-03-07T03:47:30.702 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~2702724,36221741~13720780,52484686~6706282,61973630~5750354,69970383~2374636,76391336~2927780,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,132162650~358865,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,186173555~1552003,191170311~4467874,200983077~344966,215593174~7128388,231680022~6718368,239075328~7391285,269734316~3905873,276596525~4800008,291429990~8035286,335656785~5248361,346163747~4187506,355622721~665560,379281246~4226248,385875968~9438180,405442075~6643934] 2024-03-07T03:47:30.702 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to 18 diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,77508972~1810144,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~2902976,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,215593174~7128388,231680022~6718368,239142063~7324550,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,389185474~6128674,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.702 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,77508972~1810144,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~2902976,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,215593174~7128388,231680022~6718368,239142063~7324550,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,389185474~6128674,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.708 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,76391336~2927780,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~2902976,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,215593174~7128388,231680022~6718368,239075328~7391285,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,385875968~9438180,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.709 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to 19 diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.709 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.715 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239075328~7391285,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.715 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to 15 diff [36221741~8159531,52484686~6706282,61973630~5522256,132162650~358865,231680022~6718368,239142063~7324550,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248] 2024-03-07T03:47:30.715 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [36221741~8159531,52484686~6706282,61973630~5522256,132162650~358865,231680022~6718368,239142063~7324550,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248] 2024-03-07T03:47:30.721 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [36221741~8159531,52484686~6706282,61973630~5522256,132162650~358865,231680022~6718368,239075328~7391285,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248] 2024-03-07T03:47:30.721 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to 16 diff [36221741~8159531,45945810~3996711,52484686~6706282,61973630~5522256,69970383~2374636,92768282~4310745,102928466~1050357,132162650~358865,175903570~4995247,200983077~344966,231680022~6718368,239142063~7324550,270019191~3346561,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248,389185474~6128674,405442075~6643934] 2024-03-07T03:47:30.721 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [36221741~8159531,45945810~3996711,52484686~6706282,61973630~5522256,69970383~2374636,92768282~4310745,102928466~1050357,132162650~358865,175903570~4995247,200983077~344966,231680022~6718368,239142063~7324550,270019191~3346561,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248,389185474~6128674,405442075~6643934] 2024-03-07T03:47:30.728 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [36221741~8159531,45945810~3996711,52484686~6706282,61973630~5522256,69970383~2374636,92768282~4310745,102928466~1050357,132162650~358865,175903570~4995247,200983077~344966,231680022~6718368,239075328~7391285,270019191~3346561,276596525~4800008,291429990~8035286,335656785~5248361,379281246~4226248,385875968~9438180,405442075~6643934] 2024-03-07T03:47:30.728 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to 17 diff [36221741~13720780,52484686~6706282,61973630~5750354,69970383~2374636,77508972~1810144,91110716~555941,92768282~4310745,102928466~1050357,132162650~358865,135670601~6428309,175903570~4995247,186173555~1552003,191170311~4467874,200983077~344966,215593174~7128388,231680022~6718368,239142063~7324550,270019191~3346561,276596525~4800008,291429990~8035286,335656785~5248361,346163747~4187506,355622721~665560,379281246~4226248,389185474~6128674,405442075~6643934] 2024-03-07T03:47:30.728 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [36221741~13720780,52484686~6706282,61973630~5750354,69970383~2374636,77508972~1810144,91110716~555941,92768282~4310745,102928466~1050357,132162650~358865,135670601~6428309,175903570~4995247,186173555~1552003,191170311~4467874,200983077~344966,215593174~7128388,231680022~6718368,239142063~7324550,270019191~3346561,276596525~4800008,291429990~8035286,335656785~5248361,346163747~4187506,355622721~665560,379281246~4226248,389185474~6128674,405442075~6643934] 2024-03-07T03:47:30.735 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [36221741~13720780,52484686~6706282,61973630~5750354,69970383~2374636,76391336~2927780,91110716~555941,92768282~4310745,102928466~1050357,132162650~358865,135670601~6428309,175903570~4995247,186173555~1552003,191170311~4467874,200983077~344966,215593174~7128388,231680022~6718368,239075328~7391285,270019191~3346561,276596525~4800008,291429990~8035286,335656785~5248361,346163747~4187506,355622721~665560,379281246~4226248,385875968~9438180,405442075~6643934] 2024-03-07T03:47:30.735 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to 18 diff [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,77508972~1810144,91110716~555941,92768282~4310745,102928466~1050357,120440526~2902976,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,215593174~7128388,231680022~6718368,239142063~7324550,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,389185474~6128674,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.735 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,77508972~1810144,91110716~555941,92768282~4310745,102928466~1050357,120440526~2902976,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,215593174~7128388,231680022~6718368,239142063~7324550,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,389185474~6128674,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.743 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,76391336~2927780,91110716~555941,92768282~4310745,102928466~1050357,120440526~2902976,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,215593174~7128388,231680022~6718368,239075328~7391285,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,385875968~9438180,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.743 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to 19 diff [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.743 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.749 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239075328~7391285,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.749 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to 16 diff [45945810~3996711,69970383~2374636,92768282~4310745,102928466~1050357,175903570~4995247,200983077~344966,270019191~3346561,389185474~6128674,405442075~6643934] 2024-03-07T03:47:30.749 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [45945810~3996711,69970383~2374636,92768282~4310745,102928466~1050357,175903570~4995247,200983077~344966,270019191~3346561,389185474~6128674,405442075~6643934] 2024-03-07T03:47:30.755 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [45945810~3996711,69970383~2374636,92768282~4310745,102928466~1050357,175903570~4995247,200983077~344966,270019191~3346561,385875968~9438180,405442075~6643934] 2024-03-07T03:47:30.755 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to 17 diff [43086952~6855569,64033340~3690644,69970383~2374636,77508972~1810144,91110716~555941,92768282~4310745,102928466~1050357,135670601~6428309,175903570~4995247,186173555~1552003,191170311~4467874,200983077~344966,215593174~7128388,270019191~3346561,346163747~4187506,355622721~665560,389185474~6128674,405442075~6643934] 2024-03-07T03:47:30.755 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [43086952~6855569,64033340~3690644,69970383~2374636,77508972~1810144,91110716~555941,92768282~4310745,102928466~1050357,135670601~6428309,175903570~4995247,186173555~1552003,191170311~4467874,200983077~344966,215593174~7128388,270019191~3346561,346163747~4187506,355622721~665560,389185474~6128674,405442075~6643934] 2024-03-07T03:47:30.762 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [43086952~6855569,64033340~3690644,69970383~2374636,76391336~2927780,91110716~555941,92768282~4310745,102928466~1050357,135670601~6428309,175903570~4995247,186173555~1552003,191170311~4467874,200983077~344966,215593174~7128388,270019191~3346561,346163747~4187506,355622721~665560,385875968~9438180,405442075~6643934] 2024-03-07T03:47:30.762 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to 18 diff [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,77508972~1810144,91110716~555941,92768282~4310745,102928466~1050357,120440526~2902976,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,215593174~7128388,270019191~3346561,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,389185474~6128674,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.762 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,77508972~1810144,91110716~555941,92768282~4310745,102928466~1050357,120440526~2902976,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,215593174~7128388,270019191~3346561,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,389185474~6128674,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.768 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,76391336~2927780,91110716~555941,92768282~4310745,102928466~1050357,120440526~2902976,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,215593174~7128388,270019191~3346561,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,385875968~9438180,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.768 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to 19 diff [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,251105401~1365052,262420310~3474447,270019191~3346561,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.768 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,251105401~1365052,262420310~3474447,270019191~3346561,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.774 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,251105401~1365052,262420310~3474447,270019191~3346561,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:30.774 INFO:tasks.workunit.client.0.smithi145.stdout:from 16 to 17 diff [43086952~4169818,64033340~3690644,77508972~1810144,91110716~555941,135670601~6428309,186173555~1552003,191170311~4467874,215593174~7128388,346163747~4187506,355622721~665560] 2024-03-07T03:47:30.774 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [43086952~4169818,64033340~3690644,77508972~1810144,91110716~555941,135670601~6428309,186173555~1552003,191170311~4467874,215593174~7128388,346163747~4187506,355622721~665560] 2024-03-07T03:47:30.781 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [43086952~4169818,64033340~3690644,76391336~2927780,91110716~555941,135670601~6428309,186173555~1552003,191170311~4467874,215593174~7128388,346163747~4187506,355622721~665560] 2024-03-07T03:47:30.781 INFO:tasks.workunit.client.0.smithi145.stdout:from 16 to 18 diff [30382848~5397162,43086952~4169818,49337602~10711919,64033340~3690644,77508972~1810144,91110716~555941,120440526~2902976,129286528~6266616,135670601~6428309,183443572~4858052,191170311~4467874,215593174~7128388,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,400236992~3711221] 2024-03-07T03:47:30.781 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,43086952~4169818,49337602~10711919,64033340~3690644,77508972~1810144,91110716~555941,120440526~2902976,129286528~6266616,135670601~6428309,183443572~4858052,191170311~4467874,215593174~7128388,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,400236992~3711221] 2024-03-07T03:47:30.788 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,43086952~4169818,49337602~10711919,64033340~3690644,76391336~2927780,91110716~555941,120440526~2902976,129286528~6266616,135670601~6428309,183443572~4858052,191170311~4467874,215593174~7128388,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,400236992~3711221] 2024-03-07T03:47:30.788 INFO:tasks.workunit.client.0.smithi145.stdout:from 16 to 19 diff [30382848~5397162,43086952~4169818,49337602~10711919,64033340~3690644,75029159~4365000,91110716~555941,120440526~4645226,129286528~6266616,135670601~6428309,183443572~4858052,191170311~4467874,214392323~8329239,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:30.788 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,43086952~4169818,49337602~10711919,64033340~3690644,75029159~4365000,91110716~555941,120440526~4645226,129286528~6266616,135670601~6428309,183443572~4858052,191170311~4467874,214392323~8329239,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:30.794 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,43086952~4169818,49337602~10711919,64033340~3690644,75029159~4365000,91110716~555941,120440526~4645226,129286528~6266616,135670601~6428309,183443572~4858052,191170311~4467874,214392323~8329239,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:30.794 INFO:tasks.workunit.client.0.smithi145.stdout:from 17 to 18 diff [30382848~5397162,49337602~10711919,120440526~2902976,129286528~6266616,183443572~4858052,309788973~14970654,364013514~6675474,400236992~3711221] 2024-03-07T03:47:30.794 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,49337602~10711919,120440526~2902976,129286528~6266616,183443572~4858052,309788973~14970654,364013514~6675474,400236992~3711221] 2024-03-07T03:47:30.801 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,49337602~10711919,120440526~2902976,129286528~6266616,183443572~4858052,309788973~14970654,364013514~6675474,400236992~3711221] 2024-03-07T03:47:30.801 INFO:tasks.workunit.client.0.smithi145.stdout:from 17 to 19 diff [30382848~5397162,49337602~10711919,75029159~4365000,120440526~4645226,129286528~6266616,183443572~4858052,214392323~7451196,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:30.801 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,49337602~10711919,75029159~4365000,120440526~4645226,129286528~6266616,183443572~4858052,214392323~7451196,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:30.807 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,49337602~10711919,75029159~4365000,120440526~4645226,129286528~6266616,183443572~4858052,214392323~7451196,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:30.807 INFO:tasks.workunit.client.0.smithi145.stdout:from 18 to 19 diff [75029159~4365000,121402667~3683085,130074465~2050454,214392323~7451196,251105401~1365052,262420310~3474447,292025710~6016132,384690666~8128019,405654356~2651569] 2024-03-07T03:47:30.807 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [75029159~4365000,121402667~3683085,130074465~2050454,214392323~7451196,251105401~1365052,262420310~3474447,292025710~6016132,384690666~8128019,405654356~2651569] 2024-03-07T03:47:30.813 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [75029159~4365000,121402667~3683085,130074465~2050454,214392323~7451196,251105401~1365052,262420310~3474447,292025710~6016132,384690666~8128019,405654356~2651569] 2024-03-07T03:47:30.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:30 smithi145 bash[79878]: cluster 2024-03-07T03:47:29.330685+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e1017: 8 total, 8 up, 8 in 2024-03-07T03:47:30.816 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:30 smithi145 bash[79878]: cluster 2024-03-07T03:47:29.626933+0000 mgr.y (mgr.34107) 1069 : cluster 0 pgmap v1711: 105 pgs: 105 active+clean; 825 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 47 MiB/s wr, 236 op/s 2024-03-07T03:47:30.816 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:30 smithi145 bash[82329]: cluster 2024-03-07T03:47:29.330685+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e1017: 8 total, 8 up, 8 in 2024-03-07T03:47:30.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:30 smithi145 bash[82329]: cluster 2024-03-07T03:47:29.626933+0000 mgr.y (mgr.34107) 1069 : cluster 0 pgmap v1711: 105 pgs: 105 active+clean; 825 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 47 MiB/s wr, 236 op/s 2024-03-07T03:47:30.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:30 smithi189 bash[66969]: cluster 2024-03-07T03:47:29.330685+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e1017: 8 total, 8 up, 8 in 2024-03-07T03:47:30.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:30 smithi189 bash[66969]: cluster 2024-03-07T03:47:29.626933+0000 mgr.y (mgr.34107) 1069 : cluster 0 pgmap v1711: 105 pgs: 105 active+clean; 825 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 47 MiB/s wr, 236 op/s 2024-03-07T03:47:31.359 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:31.359 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:31.397 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:31.397 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:31.397 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:31.409 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:31.409 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:31.409 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:31.422 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:31.422 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:31.422 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:31.433 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:31.434 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:31.434 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:31.443 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:31.444 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:31.444 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:31.452 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:31.452 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:31.452 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:31.461 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:31.461 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:31.461 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:31.468 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:31.468 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:31.468 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:31.475 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:31.476 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:31.476 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:31.482 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:31.483 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:31.483 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:31.489 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:31.489 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:31.489 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:31.496 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:31.496 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to HEAD diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:31.496 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:31.506 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,79691776~10313460,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:31.506 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to HEAD diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:31.506 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:31.512 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239075328~7391285,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:31.513 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to HEAD diff [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:31.513 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:31.519 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239075328~7391285,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:31.519 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to HEAD diff [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,251105401~1365052,262420310~3474447,270019191~3346561,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:31.519 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,251105401~1365052,262420310~3474447,270019191~3346561,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:31.525 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,251105401~1365052,262420310~3474447,270019191~3346561,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:31.525 INFO:tasks.workunit.client.0.smithi145.stdout:from 16 to HEAD diff [30382848~5397162,43086952~4169818,49337602~10711919,64033340~3690644,75029159~4365000,91110716~555941,120440526~4645226,129286528~6266616,135670601~6428309,183443572~4858052,191170311~4467874,214392323~8329239,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:31.525 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,43086952~4169818,49337602~10711919,64033340~3690644,75029159~4365000,91110716~555941,120440526~4645226,129286528~6266616,135670601~6428309,183443572~4858052,191170311~4467874,214392323~8329239,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:31.531 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,43086952~4169818,49337602~10711919,64033340~3690644,75029159~4365000,91110716~555941,120440526~4645226,129286528~6266616,135670601~6428309,183443572~4858052,191170311~4467874,214392323~8329239,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:31.531 INFO:tasks.workunit.client.0.smithi145.stdout:from 17 to HEAD diff [30382848~5397162,49337602~10711919,75029159~4365000,120440526~4645226,129286528~6266616,183443572~4858052,214392323~7451196,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:31.532 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,49337602~10711919,75029159~4365000,120440526~4645226,129286528~6266616,183443572~4858052,214392323~7451196,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:31.537 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,49337602~10711919,75029159~4365000,120440526~4645226,129286528~6266616,183443572~4858052,214392323~7451196,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:32.382 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:32.382 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:32.418 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:32.419 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:32.419 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:32.432 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:32.432 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:32.432 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:32.444 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:32.444 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:32.444 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:32.454 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:32.454 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:32.454 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:32.463 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:32.463 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:32.463 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:32.472 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:32.472 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:32.473 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:32.480 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:32.480 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:32.480 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:32.488 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:32.488 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:32.488 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:32.495 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:32.495 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:32.495 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:32.502 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:32.502 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:32.502 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:32.508 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:32.509 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:32.509 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:32.515 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:32.515 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to HEAD diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:32.515 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:32.523 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,79691776~10313460,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:32.524 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to HEAD diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:32.524 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:32.528 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239075328~7391285,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:32.528 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to HEAD diff [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:32.528 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:32.535 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239075328~7391285,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:32.535 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to HEAD diff [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,251105401~1365052,262420310~3474447,270019191~3346561,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:32.535 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,251105401~1365052,262420310~3474447,270019191~3346561,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:32.542 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,251105401~1365052,262420310~3474447,270019191~3346561,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:32.542 INFO:tasks.workunit.client.0.smithi145.stdout:from 16 to HEAD diff [30382848~5397162,43086952~4169818,49337602~10711919,64033340~3690644,75029159~4365000,91110716~555941,120440526~4645226,129286528~6266616,135670601~6428309,183443572~4858052,191170311~4467874,214392323~8329239,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:32.542 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,43086952~4169818,49337602~10711919,64033340~3690644,75029159~4365000,91110716~555941,120440526~4645226,129286528~6266616,135670601~6428309,183443572~4858052,191170311~4467874,214392323~8329239,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:32.548 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,43086952~4169818,49337602~10711919,64033340~3690644,75029159~4365000,91110716~555941,120440526~4645226,129286528~6266616,135670601~6428309,183443572~4858052,191170311~4467874,214392323~8329239,251105401~1365052,262420310~3474447,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~8128019,400236992~3711221,405654356~2651569] 2024-03-07T03:47:32.587 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:47:32 smithi189 bash[59814]: ts=2024-03-07T03:47:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:47:32.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:32 smithi189 bash[66969]: cluster 2024-03-07T03:47:31.354557+0000 mon.a (mon.0) 1853 : cluster 0 osdmap e1018: 8 total, 8 up, 8 in 2024-03-07T03:47:32.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:32 smithi189 bash[66969]: cluster 2024-03-07T03:47:31.627654+0000 mgr.y (mgr.34107) 1070 : cluster 0 pgmap v1713: 105 pgs: 105 active+clean; 895 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 280 KiB/s rd, 33 MiB/s wr, 2.28k op/s 2024-03-07T03:47:32.759 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:32 smithi145 bash[79878]: cluster 2024-03-07T03:47:31.354557+0000 mon.a (mon.0) 1853 : cluster 0 osdmap e1018: 8 total, 8 up, 8 in 2024-03-07T03:47:32.759 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:32 smithi145 bash[79878]: cluster 2024-03-07T03:47:31.627654+0000 mgr.y (mgr.34107) 1070 : cluster 0 pgmap v1713: 105 pgs: 105 active+clean; 895 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 280 KiB/s rd, 33 MiB/s wr, 2.28k op/s 2024-03-07T03:47:32.759 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:32 smithi145 bash[82329]: cluster 2024-03-07T03:47:31.354557+0000 mon.a (mon.0) 1853 : cluster 0 osdmap e1018: 8 total, 8 up, 8 in 2024-03-07T03:47:32.759 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:32 smithi145 bash[82329]: cluster 2024-03-07T03:47:31.627654+0000 mgr.y (mgr.34107) 1070 : cluster 0 pgmap v1713: 105 pgs: 105 active+clean; 895 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 280 KiB/s rd, 33 MiB/s wr, 2.28k op/s 2024-03-07T03:47:33.066 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:47:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:47:32] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.43.0" 2024-03-07T03:47:33.389 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:33.389 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:33.428 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:33.428 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:33.428 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:33.439 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:33.439 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:33.439 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:33.449 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:33.449 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:33.449 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:33.458 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:33.458 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:33.458 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:33.467 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:33.467 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:33.467 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:33.474 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:33.475 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:33.475 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:33.482 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:33.482 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:33.482 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:33.489 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:33.489 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:33.489 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:33.495 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:33.495 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:33.495 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:33.503 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:33.503 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:33.503 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:33.515 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:33.515 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:33.515 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:33.521 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:33.521 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to HEAD diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:33.521 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:33.527 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,79691776~10313460,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:33.527 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to HEAD diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:33.527 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:33.534 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239075328~7391285,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:33.534 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to HEAD diff [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:33.535 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:33.541 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239075328~7391285,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:33.541 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to HEAD diff [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,251105401~1365052,262420310~3474447,270019191~3346561,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:33.542 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,251105401~1365052,262420310~3474447,270019191~3346561,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:33.547 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,43086952~16962569,64033340~3690644,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,251105401~1365052,262420310~3474447,270019191~3346561,292025710~6016132,309788973~14970654,346163747~4187506,355622721~665560,364013514~6675474,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:33.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:33 smithi145 bash[79878]: cluster 2024-03-07T03:47:32.362007+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e1019: 8 total, 8 up, 8 in 2024-03-07T03:47:33.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:33 smithi145 bash[79878]: audit 2024-03-07T03:47:32.680793+0000 mon.a (mon.0) 1855 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:33.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:33 smithi145 bash[82329]: cluster 2024-03-07T03:47:32.362007+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e1019: 8 total, 8 up, 8 in 2024-03-07T03:47:33.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:33 smithi145 bash[82329]: audit 2024-03-07T03:47:32.680793+0000 mon.a (mon.0) 1855 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:33.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:33 smithi189 bash[66969]: cluster 2024-03-07T03:47:32.362007+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e1019: 8 total, 8 up, 8 in 2024-03-07T03:47:33.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:33 smithi189 bash[66969]: audit 2024-03-07T03:47:32.680793+0000 mon.a (mon.0) 1855 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:34.421 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:34.422 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:34.450 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:34.451 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:34.451 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:34.461 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:34.462 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:34.462 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:34.471 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:34.471 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:34.471 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:34.480 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:34.480 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:34.480 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:34.488 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:34.488 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:34.488 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:34.495 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:34.496 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:34.496 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:34.503 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:34.503 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:34.503 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:34.510 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:34.510 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:34.510 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:34.517 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:34.517 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:34.517 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:34.524 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:34.524 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:34.524 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:34.530 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:34.531 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:34.531 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:34.537 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:34.537 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to HEAD diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:34.537 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:34.544 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,79691776~10313460,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:34.544 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to HEAD diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:34.544 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:34.551 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239075328~7391285,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:34.551 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to HEAD diff [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:34.551 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:34.558 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [30382848~5397162,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6428309,175903570~4995247,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239075328~7391285,251105401~1365052,262420310~3474447,270019191~3346561,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:34.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:34 smithi145 bash[79878]: cluster 2024-03-07T03:47:33.365199+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e1020: 8 total, 8 up, 8 in 2024-03-07T03:47:34.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:34 smithi145 bash[79878]: cluster 2024-03-07T03:47:33.628684+0000 mgr.y (mgr.34107) 1071 : cluster 0 pgmap v1716: 105 pgs: 105 active+clean; 904 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 175 KiB/s rd, 19 MiB/s wr, 4.77k op/s 2024-03-07T03:47:34.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:34 smithi145 bash[82329]: cluster 2024-03-07T03:47:33.365199+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e1020: 8 total, 8 up, 8 in 2024-03-07T03:47:34.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:34 smithi145 bash[82329]: cluster 2024-03-07T03:47:33.628684+0000 mgr.y (mgr.34107) 1071 : cluster 0 pgmap v1716: 105 pgs: 105 active+clean; 904 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 175 KiB/s rd, 19 MiB/s wr, 4.77k op/s 2024-03-07T03:47:34.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:34 smithi189 bash[66969]: cluster 2024-03-07T03:47:33.365199+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e1020: 8 total, 8 up, 8 in 2024-03-07T03:47:34.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:34 smithi189 bash[66969]: cluster 2024-03-07T03:47:33.628684+0000 mgr.y (mgr.34107) 1071 : cluster 0 pgmap v1716: 105 pgs: 105 active+clean; 904 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 175 KiB/s rd, 19 MiB/s wr, 4.77k op/s 2024-03-07T03:47:35.416 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:35.417 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:35.454 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:35.454 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:35.455 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:35.464 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:35.464 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:35.464 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:35.473 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:35.473 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:35.473 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:35.482 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:35.482 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:35.483 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:35.490 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:35.490 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:35.490 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:35.497 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:35.497 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:35.497 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:35.504 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:35.504 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:35.505 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:35.511 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:35.512 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:35.512 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:35.518 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:35.518 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:35.518 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:35.524 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:35.524 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:35.524 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:35.531 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:35.531 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:35.531 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:35.537 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:35.537 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to HEAD diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:35.537 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:35.544 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,79691776~10313460,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:35.544 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to HEAD diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:35.544 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239142063~7324550,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:35.550 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~5750354,69970383~2374636,75029159~4365000,84122472~5882764,91110716~555941,92768282~4310745,102928466~1050357,120440526~4645226,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,200983077~344966,214392323~8329239,231680022~6718368,239075328~7391285,251105401~1365052,262420310~3474447,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~10623482,400236992~3711221,405442075~6643934] 2024-03-07T03:47:35.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:35 smithi145 bash[79878]: cluster 2024-03-07T03:47:34.386681+0000 mon.a (mon.0) 1857 : cluster 0 osdmap e1021: 8 total, 8 up, 8 in 2024-03-07T03:47:35.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:35 smithi145 bash[82329]: cluster 2024-03-07T03:47:34.386681+0000 mon.a (mon.0) 1857 : cluster 0 osdmap e1021: 8 total, 8 up, 8 in 2024-03-07T03:47:35.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:35 smithi189 bash[66969]: cluster 2024-03-07T03:47:34.386681+0000 mon.a (mon.0) 1857 : cluster 0 osdmap e1021: 8 total, 8 up, 8 in 2024-03-07T03:47:36.436 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:36.436 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:36.478 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:36.478 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:36.478 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:36.487 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:36.487 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:36.487 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:36.496 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:36.496 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:36.496 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:36.504 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:36.504 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:36.504 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:36.511 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:36.511 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:36.511 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:36.518 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:36.518 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:36.518 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:36.524 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:36.524 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:36.525 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:36.531 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:36.531 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:36.531 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:36.538 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:36.538 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:36.538 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:36.544 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:36.544 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:36.544 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:36.551 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:36.551 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:36.551 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:36.557 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:36.557 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to HEAD diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:36.558 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,82227694~7777542,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,145598567~2134643,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:36.565 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [27711596~8068414,36221741~23827780,61973630~7150629,69970383~2374636,75029159~4365000,79691776~10313460,91110716~555941,92768282~4564932,98329964~7380989,117334964~7750788,129286528~6266616,135670601~6575091,142606336~5126874,151434756~8115093,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,379281246~4226248,384690666~19257547,405442075~6643934] 2024-03-07T03:47:36.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:36 smithi145 bash[79878]: cluster 2024-03-07T03:47:35.390039+0000 mon.a (mon.0) 1858 : cluster 0 osdmap e1022: 8 total, 8 up, 8 in 2024-03-07T03:47:36.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:36 smithi145 bash[79878]: cluster 2024-03-07T03:47:35.629391+0000 mgr.y (mgr.34107) 1072 : cluster 0 pgmap v1719: 105 pgs: 105 active+clean; 901 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 135 KiB/s rd, 3.2 MiB/s wr, 3.59k op/s 2024-03-07T03:47:36.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:36 smithi145 bash[82329]: cluster 2024-03-07T03:47:35.390039+0000 mon.a (mon.0) 1858 : cluster 0 osdmap e1022: 8 total, 8 up, 8 in 2024-03-07T03:47:36.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:36 smithi145 bash[82329]: cluster 2024-03-07T03:47:35.629391+0000 mgr.y (mgr.34107) 1072 : cluster 0 pgmap v1719: 105 pgs: 105 active+clean; 901 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 135 KiB/s rd, 3.2 MiB/s wr, 3.59k op/s 2024-03-07T03:47:36.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:36 smithi189 bash[66969]: cluster 2024-03-07T03:47:35.390039+0000 mon.a (mon.0) 1858 : cluster 0 osdmap e1022: 8 total, 8 up, 8 in 2024-03-07T03:47:36.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:36 smithi189 bash[66969]: cluster 2024-03-07T03:47:35.629391+0000 mgr.y (mgr.34107) 1072 : cluster 0 pgmap v1719: 105 pgs: 105 active+clean; 901 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 135 KiB/s rd, 3.2 MiB/s wr, 3.59k op/s 2024-03-07T03:47:37.456 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:37.456 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:37.487 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:37.487 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:37.487 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:37.496 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:37.496 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:37.496 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:37.503 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:37.504 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:37.504 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:37.511 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:37.511 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:37.511 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:37.518 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:37.518 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:37.518 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:37.525 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:37.525 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:37.525 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:37.531 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:37.532 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:37.532 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:37.538 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:37.538 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:37.538 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:37.545 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:37.545 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:37.545 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:37.551 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:37.551 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:37.551 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:37.557 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:37.557 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:37.557 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:37.564 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,291429990~8035286,309788973~14970654,335656785~5248361,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:37.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:37 smithi145 bash[79878]: cluster 2024-03-07T03:47:36.418774+0000 mon.a (mon.0) 1859 : cluster 0 osdmap e1023: 8 total, 8 up, 8 in 2024-03-07T03:47:37.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:37 smithi145 bash[82329]: cluster 2024-03-07T03:47:36.418774+0000 mon.a (mon.0) 1859 : cluster 0 osdmap e1023: 8 total, 8 up, 8 in 2024-03-07T03:47:37.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:37 smithi189 bash[66969]: cluster 2024-03-07T03:47:36.418774+0000 mon.a (mon.0) 1859 : cluster 0 osdmap e1023: 8 total, 8 up, 8 in 2024-03-07T03:47:38.481 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:38.481 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:38.505 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:38.505 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:38.505 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:38.513 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:38.513 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:38.513 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:38.520 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:38.520 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:38.520 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:38.527 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:38.528 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:38.528 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:38.535 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:38.535 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:38.535 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:38.542 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:38.542 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:38.542 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:38.548 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:38.548 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:38.548 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:38.555 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:38.555 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:38.555 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:38.561 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:38.561 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:38.561 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:38.567 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:38.567 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:38.567 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:38.574 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114766453~10319299,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~4662106,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,287416788~2430416,291429990~8244131,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,364013514~6675474,378511893~4995601,384690666~27395343] 2024-03-07T03:47:38.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:38 smithi145 bash[79878]: cluster 2024-03-07T03:47:37.411927+0000 mon.a (mon.0) 1860 : cluster 0 osdmap e1024: 8 total, 8 up, 8 in 2024-03-07T03:47:38.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:38 smithi145 bash[79878]: cluster 2024-03-07T03:47:37.630043+0000 mgr.y (mgr.34107) 1073 : cluster 0 pgmap v1722: 105 pgs: 105 active+clean; 871 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.7 KiB/s wr, 1.76k op/s 2024-03-07T03:47:38.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:38 smithi145 bash[82329]: cluster 2024-03-07T03:47:37.411927+0000 mon.a (mon.0) 1860 : cluster 0 osdmap e1024: 8 total, 8 up, 8 in 2024-03-07T03:47:38.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:38 smithi145 bash[82329]: cluster 2024-03-07T03:47:37.630043+0000 mgr.y (mgr.34107) 1073 : cluster 0 pgmap v1722: 105 pgs: 105 active+clean; 871 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.7 KiB/s wr, 1.76k op/s 2024-03-07T03:47:38.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:38 smithi189 bash[66969]: cluster 2024-03-07T03:47:37.411927+0000 mon.a (mon.0) 1860 : cluster 0 osdmap e1024: 8 total, 8 up, 8 in 2024-03-07T03:47:38.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:38 smithi189 bash[66969]: cluster 2024-03-07T03:47:37.630043+0000 mgr.y (mgr.34107) 1073 : cluster 0 pgmap v1722: 105 pgs: 105 active+clean; 871 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.7 KiB/s wr, 1.76k op/s 2024-03-07T03:47:39.479 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:39.479 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:39.509 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:39.509 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:39.509 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:39.517 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:39.517 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:39.517 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:39.525 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:39.525 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:39.525 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:39.532 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:39.532 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:39.532 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:39.539 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:39.539 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:39.539 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:39.545 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:39.546 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:39.546 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:39.552 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:39.552 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:39.552 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:39.558 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:39.558 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:39.558 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:39.565 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:39.565 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:39.565 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~8068414,36221741~23827780,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:39.571 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~8068414,36221741~23827780,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~17746333,183443572~4858052,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~665560,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:39.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:39 smithi145 bash[79878]: cluster 2024-03-07T03:47:38.432975+0000 mon.a (mon.0) 1861 : cluster 0 osdmap e1025: 8 total, 8 up, 8 in 2024-03-07T03:47:39.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:39 smithi145 bash[82329]: cluster 2024-03-07T03:47:38.432975+0000 mon.a (mon.0) 1861 : cluster 0 osdmap e1025: 8 total, 8 up, 8 in 2024-03-07T03:47:39.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:39 smithi189 bash[66969]: cluster 2024-03-07T03:47:38.432975+0000 mon.a (mon.0) 1861 : cluster 0 osdmap e1025: 8 total, 8 up, 8 in 2024-03-07T03:47:40.497 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:40.497 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:40.537 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:40.537 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:40.538 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:40.546 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:40.546 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:40.547 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:40.554 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:40.554 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:40.554 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:40.562 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:40.562 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:40.562 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:40.569 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:40.569 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:40.569 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:40.575 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:40.576 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:40.576 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:40.582 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:40.582 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:40.582 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:40.588 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:40.589 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:40.589 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~7777542,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:40.595 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~10313460,91110716~555941,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163152484~25481007,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:40.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:40 smithi145 bash[79878]: cluster 2024-03-07T03:47:39.433633+0000 mon.a (mon.0) 1862 : cluster 0 osdmap e1026: 8 total, 8 up, 8 in 2024-03-07T03:47:40.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:40 smithi145 bash[79878]: cluster 2024-03-07T03:47:39.630676+0000 mgr.y (mgr.34107) 1074 : cluster 0 pgmap v1725: 105 pgs: 105 active+clean; 833 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.2 KiB/s wr, 1.63k op/s 2024-03-07T03:47:40.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:40 smithi145 bash[82329]: cluster 2024-03-07T03:47:39.433633+0000 mon.a (mon.0) 1862 : cluster 0 osdmap e1026: 8 total, 8 up, 8 in 2024-03-07T03:47:40.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:40 smithi145 bash[82329]: cluster 2024-03-07T03:47:39.630676+0000 mgr.y (mgr.34107) 1074 : cluster 0 pgmap v1725: 105 pgs: 105 active+clean; 833 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.2 KiB/s wr, 1.63k op/s 2024-03-07T03:47:40.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:40 smithi189 bash[66969]: cluster 2024-03-07T03:47:39.433633+0000 mon.a (mon.0) 1862 : cluster 0 osdmap e1026: 8 total, 8 up, 8 in 2024-03-07T03:47:40.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:40 smithi189 bash[66969]: cluster 2024-03-07T03:47:39.630676+0000 mgr.y (mgr.34107) 1074 : cluster 0 pgmap v1725: 105 pgs: 105 active+clean; 833 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.2 KiB/s wr, 1.63k op/s 2024-03-07T03:47:41.511 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:41.511 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:41.551 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:41.552 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:41.552 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:41.558 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:41.558 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:41.558 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:41.568 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:41.568 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:41.568 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:41.576 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:41.576 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:41.576 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:41.584 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:41.584 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:41.584 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:41.591 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:41.591 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:41.591 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:41.597 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:41.598 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:41.598 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [11633950~3602920,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~2134643,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~7324550,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:41.604 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [11633950~3602920,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~5126874,147989099~11560750,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~7391285,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~938793,358406858~1989264,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:41.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:41 smithi145 bash[79878]: cluster 2024-03-07T03:47:40.465594+0000 mon.a (mon.0) 1863 : cluster 0 osdmap e1027: 8 total, 8 up, 8 in 2024-03-07T03:47:41.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:41 smithi145 bash[82329]: cluster 2024-03-07T03:47:40.465594+0000 mon.a (mon.0) 1863 : cluster 0 osdmap e1027: 8 total, 8 up, 8 in 2024-03-07T03:47:41.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:41 smithi189 bash[66969]: cluster 2024-03-07T03:47:40.465594+0000 mon.a (mon.0) 1863 : cluster 0 osdmap e1027: 8 total, 8 up, 8 in 2024-03-07T03:47:42.525 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:42.525 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:42.572 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:42.572 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:42.572 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:42.579 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:42.579 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:42.579 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:42.587 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:42.587 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:42.587 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:42.587 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:47:42 smithi189 bash[59814]: ts=2024-03-07T03:47:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:47:42.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:42 smithi189 bash[66969]: cluster 2024-03-07T03:47:41.473229+0000 mon.a (mon.0) 1864 : cluster 0 osdmap e1028: 8 total, 8 up, 8 in 2024-03-07T03:47:42.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:42 smithi189 bash[66969]: cluster 2024-03-07T03:47:41.631202+0000 mgr.y (mgr.34107) 1075 : cluster 0 pgmap v1728: 105 pgs: 105 active+clean; 784 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 1.42k op/s 2024-03-07T03:47:42.594 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:42.594 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:42.594 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:42.601 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:42.601 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:42.601 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:42.608 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:42.608 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:42.608 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [8700349~6536521,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,375226132~8281362,384690666~27395343] 2024-03-07T03:47:42.614 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [8700349~6536521,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,129286528~12959164,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,224056275~91427,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,306214394~1374272,309137334~15667515,335656785~10126933,346163747~4187506,355622721~5759835,364013514~6675474,373293056~10214438,384690666~27395343] 2024-03-07T03:47:42.759 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:42 smithi145 bash[79878]: cluster 2024-03-07T03:47:41.473229+0000 mon.a (mon.0) 1864 : cluster 0 osdmap e1028: 8 total, 8 up, 8 in 2024-03-07T03:47:42.759 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:42 smithi145 bash[79878]: cluster 2024-03-07T03:47:41.631202+0000 mgr.y (mgr.34107) 1075 : cluster 0 pgmap v1728: 105 pgs: 105 active+clean; 784 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 1.42k op/s 2024-03-07T03:47:42.759 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:42 smithi145 bash[82329]: cluster 2024-03-07T03:47:41.473229+0000 mon.a (mon.0) 1864 : cluster 0 osdmap e1028: 8 total, 8 up, 8 in 2024-03-07T03:47:42.759 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:42 smithi145 bash[82329]: cluster 2024-03-07T03:47:41.631202+0000 mgr.y (mgr.34107) 1075 : cluster 0 pgmap v1728: 105 pgs: 105 active+clean; 784 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 1.42k op/s 2024-03-07T03:47:43.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:47:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:47:42] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.43.0" 2024-03-07T03:47:43.141 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:43.141 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:43.176 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:43.176 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:43.177 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:43.184 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:43.185 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:43.185 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:43.192 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:43.192 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:43.192 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:43.199 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:43.199 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:43.199 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:43.206 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:43.206 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:43.206 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,375226132~8281362,384690666~28202737] 2024-03-07T03:47:43.212 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,335656785~10126933,346163747~8829275,355622721~5759835,364013514~6675474,373293056~10214438,384690666~28202737] 2024-03-07T03:47:43.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:43 smithi145 bash[79878]: cluster 2024-03-07T03:47:42.501467+0000 mon.a (mon.0) 1865 : cluster 0 osdmap e1029: 8 total, 8 up, 8 in 2024-03-07T03:47:43.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:43 smithi145 bash[79878]: cluster 2024-03-07T03:47:43.108024+0000 mon.a (mon.0) 1866 : cluster 0 osdmap e1030: 8 total, 8 up, 8 in 2024-03-07T03:47:43.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:43 smithi145 bash[82329]: cluster 2024-03-07T03:47:42.501467+0000 mon.a (mon.0) 1865 : cluster 0 osdmap e1029: 8 total, 8 up, 8 in 2024-03-07T03:47:43.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:43 smithi145 bash[82329]: cluster 2024-03-07T03:47:43.108024+0000 mon.a (mon.0) 1866 : cluster 0 osdmap e1030: 8 total, 8 up, 8 in 2024-03-07T03:47:43.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:43 smithi189 bash[66969]: cluster 2024-03-07T03:47:42.501467+0000 mon.a (mon.0) 1865 : cluster 0 osdmap e1029: 8 total, 8 up, 8 in 2024-03-07T03:47:43.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:43 smithi189 bash[66969]: cluster 2024-03-07T03:47:43.108024+0000 mon.a (mon.0) 1866 : cluster 0 osdmap e1030: 8 total, 8 up, 8 in 2024-03-07T03:47:44.130 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:44.130 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:44.186 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:44.186 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:44.186 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:44.195 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:44.196 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:44.196 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:44.207 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:44.207 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:44.207 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:44.212 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:44.212 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:44.212 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~13753900,114533580~10552172,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~11558560,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,304833665~19971184,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:44.220 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~13753900,114533580~10552172,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~11625295,251105401~6945683,259345944~6967198,269734316~3905873,276596525~4800008,285480198~14193923,301989888~22814961,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:45.156 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:45.156 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:45.182 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:45.182 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:45.182 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:45.190 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:45.190 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:45.190 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:45.201 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:45.201 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:45.201 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,61973630~17420529,82227694~10343148,92768282~4564932,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~4800008,284525397~15148724,304833665~19971184,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,375226132~8281362,384690666~28202737] 2024-03-07T03:47:45.208 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~32337925,61973630~17420529,79691776~12879066,92768282~4564932,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~4800008,283906311~15767810,301989888~22814961,329627709~325719,334175736~20817286,355622721~5759835,362225914~8463074,373293056~10214438,384690666~28202737] 2024-03-07T03:47:45.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:45 smithi145 bash[79878]: cluster 2024-03-07T03:47:43.632064+0000 mgr.y (mgr.34107) 1076 : cluster 0 pgmap v1731: 105 pgs: 105 active+clean; 707 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s rd, 6.0 KiB/s wr, 1.40k op/s 2024-03-07T03:47:45.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:45 smithi145 bash[79878]: cluster 2024-03-07T03:47:44.108198+0000 mon.a (mon.0) 1867 : cluster 0 osdmap e1031: 8 total, 8 up, 8 in 2024-03-07T03:47:45.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:45 smithi145 bash[82329]: cluster 2024-03-07T03:47:43.632064+0000 mgr.y (mgr.34107) 1076 : cluster 0 pgmap v1731: 105 pgs: 105 active+clean; 707 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s rd, 6.0 KiB/s wr, 1.40k op/s 2024-03-07T03:47:45.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:45 smithi145 bash[82329]: cluster 2024-03-07T03:47:44.108198+0000 mon.a (mon.0) 1867 : cluster 0 osdmap e1031: 8 total, 8 up, 8 in 2024-03-07T03:47:45.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:45 smithi189 bash[66969]: cluster 2024-03-07T03:47:43.632064+0000 mgr.y (mgr.34107) 1076 : cluster 0 pgmap v1731: 105 pgs: 105 active+clean; 707 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s rd, 6.0 KiB/s wr, 1.40k op/s 2024-03-07T03:47:45.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:45 smithi189 bash[66969]: cluster 2024-03-07T03:47:44.108198+0000 mon.a (mon.0) 1867 : cluster 0 osdmap e1031: 8 total, 8 up, 8 in 2024-03-07T03:47:46.177 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:46.178 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:46.210 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:46.210 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:46.210 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:46.219 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:46.219 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:46.219 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4327490~14474229,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,98329964~26755788,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~18909021,259345944~6967198,269734316~3905873,276596525~7309786,284525397~15148724,304833665~19971184,329627709~325719,331086294~30296262,362225914~9431268,375226132~8281362,384690666~28202737] 2024-03-07T03:47:46.228 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4327490~14474229,20971520~5600338,27711596~51682563,79691776~17641438,98329964~26755788,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~9828336,227584913~2698758,231680022~6718368,239075328~18975756,259345944~6967198,269734316~3905873,276596525~23077596,301989888~22814961,327420289~44236893,373293056~10214438,384690666~28202737] 2024-03-07T03:47:46.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:46 smithi145 bash[79878]: cluster 2024-03-07T03:47:45.114959+0000 mon.a (mon.0) 1868 : cluster 0 osdmap e1032: 8 total, 8 up, 8 in 2024-03-07T03:47:46.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:46 smithi145 bash[82329]: cluster 2024-03-07T03:47:45.114959+0000 mon.a (mon.0) 1868 : cluster 0 osdmap e1032: 8 total, 8 up, 8 in 2024-03-07T03:47:46.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:46 smithi189 bash[66969]: cluster 2024-03-07T03:47:45.114959+0000 mon.a (mon.0) 1868 : cluster 0 osdmap e1032: 8 total, 8 up, 8 in 2024-03-07T03:47:47.169 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:47.169 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:47.204 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:47.204 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:47.204 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~13951282,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,214392323~8329239,223103880~1116779,227584913~2698758,231680022~6718368,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:47.212 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~16943513,163070735~25562756,190129935~302950,191170311~4467874,198381649~5035730,206070966~7558505,213909504~10311155,227584913~2698758,231680022~6718368,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:47.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:47 smithi145 bash[79878]: cluster 2024-03-07T03:47:45.632727+0000 mgr.y (mgr.34107) 1077 : cluster 0 pgmap v1734: 105 pgs: 105 active+clean; 705 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 6.0 KiB/s wr, 777 op/s 2024-03-07T03:47:47.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:47 smithi145 bash[79878]: cluster 2024-03-07T03:47:46.141704+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e1033: 8 total, 8 up, 8 in 2024-03-07T03:47:47.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:47 smithi145 bash[82329]: cluster 2024-03-07T03:47:45.632727+0000 mgr.y (mgr.34107) 1077 : cluster 0 pgmap v1734: 105 pgs: 105 active+clean; 705 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 6.0 KiB/s wr, 777 op/s 2024-03-07T03:47:47.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:47 smithi145 bash[82329]: cluster 2024-03-07T03:47:46.141704+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e1033: 8 total, 8 up, 8 in 2024-03-07T03:47:47.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:47 smithi189 bash[66969]: cluster 2024-03-07T03:47:45.632727+0000 mgr.y (mgr.34107) 1077 : cluster 0 pgmap v1734: 105 pgs: 105 active+clean; 705 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 6.0 KiB/s wr, 777 op/s 2024-03-07T03:47:47.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:47 smithi189 bash[66969]: cluster 2024-03-07T03:47:46.141704+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e1033: 8 total, 8 up, 8 in 2024-03-07T03:47:48.146 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:48.146 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [2318850~16482869,24990107~1581751,27711596~32337925,60790717~18603442,82227694~15105520,97961740~27124012,126185278~16060414,145598567~43034924,190129935~302950,191170311~12247068,206070966~7558505,214392323~8329239,223103880~1116779,226998752~11399638,239142063~34498126,276596525~7309786,284525397~15148724,304833665~22586624,329627709~325719,331086294~30296262,362225914~9431268,375226132~37667271] 2024-03-07T03:47:48.184 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~18801719,20971520~5600338,27711596~51682563,79691776~45393976,125829120~16416572,142606336~46027155,188743680~14673699,206070966~7558505,213909504~10311155,226492416~11905974,239075328~34564861,276596525~23077596,301989888~69667294,373293056~39600347] 2024-03-07T03:47:48.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:48 smithi145 bash[79878]: cluster 2024-03-07T03:47:47.142313+0000 mon.a (mon.0) 1870 : cluster 0 osdmap e1034: 8 total, 8 up, 8 in 2024-03-07T03:47:48.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:48 smithi145 bash[79878]: cluster 2024-03-07T03:47:47.633421+0000 mgr.y (mgr.34107) 1078 : cluster 0 pgmap v1737: 105 pgs: 105 active+clean; 621 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 5.0 KiB/s wr, 573 op/s 2024-03-07T03:47:48.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:48 smithi145 bash[79878]: audit 2024-03-07T03:47:47.681100+0000 mon.a (mon.0) 1871 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:48.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:48 smithi145 bash[79878]: cluster 2024-03-07T03:47:48.111348+0000 mon.a (mon.0) 1872 : cluster 0 osdmap e1035: 8 total, 8 up, 8 in 2024-03-07T03:47:48.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:48 smithi145 bash[82329]: cluster 2024-03-07T03:47:47.142313+0000 mon.a (mon.0) 1870 : cluster 0 osdmap e1034: 8 total, 8 up, 8 in 2024-03-07T03:47:48.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:48 smithi145 bash[82329]: cluster 2024-03-07T03:47:47.633421+0000 mgr.y (mgr.34107) 1078 : cluster 0 pgmap v1737: 105 pgs: 105 active+clean; 621 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 5.0 KiB/s wr, 573 op/s 2024-03-07T03:47:48.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:48 smithi145 bash[82329]: audit 2024-03-07T03:47:47.681100+0000 mon.a (mon.0) 1871 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:48.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:48 smithi145 bash[82329]: cluster 2024-03-07T03:47:48.111348+0000 mon.a (mon.0) 1872 : cluster 0 osdmap e1035: 8 total, 8 up, 8 in 2024-03-07T03:47:48.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:48 smithi189 bash[66969]: cluster 2024-03-07T03:47:47.142313+0000 mon.a (mon.0) 1870 : cluster 0 osdmap e1034: 8 total, 8 up, 8 in 2024-03-07T03:47:48.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:48 smithi189 bash[66969]: cluster 2024-03-07T03:47:47.633421+0000 mgr.y (mgr.34107) 1078 : cluster 0 pgmap v1737: 105 pgs: 105 active+clean; 621 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 5.0 KiB/s wr, 573 op/s 2024-03-07T03:47:48.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:48 smithi189 bash[66969]: audit 2024-03-07T03:47:47.681100+0000 mon.a (mon.0) 1871 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:48.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:48 smithi189 bash[66969]: cluster 2024-03-07T03:47:48.111348+0000 mon.a (mon.0) 1872 : cluster 0 osdmap e1035: 8 total, 8 up, 8 in 2024-03-07T03:47:49.224 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/0.DiffIterateStress (40070 ms) 2024-03-07T03:47:49.224 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParent 2024-03-07T03:47:49.225 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:47:50.440 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/0.DiffIterateParent (1216 ms) 2024-03-07T03:47:50.440 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/0.DiffIterateIgnoreParent 2024-03-07T03:47:50.440 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:47:50.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:50 smithi145 bash[79878]: cluster 2024-03-07T03:47:49.172904+0000 mon.a (mon.0) 1873 : cluster 0 osdmap e1036: 8 total, 8 up, 8 in 2024-03-07T03:47:50.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:50 smithi145 bash[79878]: cluster 2024-03-07T03:47:49.634098+0000 mgr.y (mgr.34107) 1079 : cluster 0 pgmap v1740: 105 pgs: 105 active+clean; 556 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 767 B/s wr, 506 op/s 2024-03-07T03:47:50.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:50 smithi145 bash[82329]: cluster 2024-03-07T03:47:49.172904+0000 mon.a (mon.0) 1873 : cluster 0 osdmap e1036: 8 total, 8 up, 8 in 2024-03-07T03:47:50.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:50 smithi145 bash[82329]: cluster 2024-03-07T03:47:49.634098+0000 mgr.y (mgr.34107) 1079 : cluster 0 pgmap v1740: 105 pgs: 105 active+clean; 556 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 767 B/s wr, 506 op/s 2024-03-07T03:47:50.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:50 smithi189 bash[66969]: cluster 2024-03-07T03:47:49.172904+0000 mon.a (mon.0) 1873 : cluster 0 osdmap e1036: 8 total, 8 up, 8 in 2024-03-07T03:47:50.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:50 smithi189 bash[66969]: cluster 2024-03-07T03:47:49.634098+0000 mgr.y (mgr.34107) 1079 : cluster 0 pgmap v1740: 105 pgs: 105 active+clean; 556 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 767 B/s wr, 506 op/s 2024-03-07T03:47:51.474 INFO:tasks.workunit.client.0.smithi145.stdout: wrote [299397~61287,802578~89997,3455956~89341,4519697~92415,8122707~71776,8429062~7231,17134160~8464,19873433~20641,19901134~84156] to clone 2024-03-07T03:47:51.475 INFO:tasks.workunit.client.0.smithi145.stdout: diff was [0~3545297,4194304~4000179,8388608~47685,16777216~3208074] 2024-03-07T03:47:51.485 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/0.DiffIterateIgnoreParent (1045 ms) 2024-03-07T03:47:51.485 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/0.DiffIterateCallbackError 2024-03-07T03:47:51.485 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:47:51.537 INFO:tasks.workunit.client.0.smithi145.stdout: wrote [142924~48246,3157727~48939,4177915~90573,5583191~87521,5693583~31999,6061935~33195,12546726~40744,12673014~48512,18829546~20780,20354763~41591] 2024-03-07T03:47:51.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:51 smithi145 bash[79878]: cluster 2024-03-07T03:47:50.184192+0000 mon.a (mon.0) 1874 : cluster 0 osdmap e1037: 8 total, 8 up, 8 in 2024-03-07T03:47:51.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:51 smithi145 bash[82329]: cluster 2024-03-07T03:47:50.184192+0000 mon.a (mon.0) 1874 : cluster 0 osdmap e1037: 8 total, 8 up, 8 in 2024-03-07T03:47:51.568 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/0.DiffIterateCallbackError (83 ms) 2024-03-07T03:47:51.568 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParentDiscard 2024-03-07T03:47:51.568 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:47:51.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:51 smithi189 bash[66969]: cluster 2024-03-07T03:47:50.184192+0000 mon.a (mon.0) 1874 : cluster 0 osdmap e1037: 8 total, 8 up, 8 in 2024-03-07T03:47:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:52 smithi145 bash[79878]: cluster 2024-03-07T03:47:51.191638+0000 mon.a (mon.0) 1875 : cluster 0 osdmap e1038: 8 total, 8 up, 8 in 2024-03-07T03:47:52.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:52 smithi145 bash[79878]: cluster 2024-03-07T03:47:51.635004+0000 mgr.y (mgr.34107) 1080 : cluster 0 pgmap v1743: 105 pgs: 105 active+clean; 503 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 3.5 MiB/s wr, 323 op/s 2024-03-07T03:47:52.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:52 smithi145 bash[82329]: cluster 2024-03-07T03:47:51.191638+0000 mon.a (mon.0) 1875 : cluster 0 osdmap e1038: 8 total, 8 up, 8 in 2024-03-07T03:47:52.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:52 smithi145 bash[82329]: cluster 2024-03-07T03:47:51.635004+0000 mgr.y (mgr.34107) 1080 : cluster 0 pgmap v1743: 105 pgs: 105 active+clean; 503 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 3.5 MiB/s wr, 323 op/s 2024-03-07T03:47:52.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:52 smithi189 bash[66969]: cluster 2024-03-07T03:47:51.191638+0000 mon.a (mon.0) 1875 : cluster 0 osdmap e1038: 8 total, 8 up, 8 in 2024-03-07T03:47:52.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:52 smithi189 bash[66969]: cluster 2024-03-07T03:47:51.635004+0000 mgr.y (mgr.34107) 1080 : cluster 0 pgmap v1743: 105 pgs: 105 active+clean; 503 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 3.5 MiB/s wr, 323 op/s 2024-03-07T03:47:52.588 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:47:52 smithi189 bash[59814]: ts=2024-03-07T03:47:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:47:53.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:47:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:47:52] "GET /metrics HTTP/1.1" 200 36590 "" "Prometheus/2.43.0" 2024-03-07T03:47:53.365 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/0.DiffIterateParentDiscard (1797 ms) 2024-03-07T03:47:53.366 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnalignedSmall 2024-03-07T03:47:53.366 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:47:53.480 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnalignedSmall (115 ms) 2024-03-07T03:47:53.480 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnaligned 2024-03-07T03:47:53.481 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:47:53.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:53 smithi145 bash[79878]: cluster 2024-03-07T03:47:52.204486+0000 mon.a (mon.0) 1876 : cluster 0 osdmap e1039: 8 total, 8 up, 8 in 2024-03-07T03:47:53.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:53 smithi145 bash[79878]: cluster 2024-03-07T03:47:53.113278+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e1040: 8 total, 8 up, 8 in 2024-03-07T03:47:53.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:53 smithi145 bash[82329]: cluster 2024-03-07T03:47:52.204486+0000 mon.a (mon.0) 1876 : cluster 0 osdmap e1039: 8 total, 8 up, 8 in 2024-03-07T03:47:53.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:53 smithi145 bash[82329]: cluster 2024-03-07T03:47:53.113278+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e1040: 8 total, 8 up, 8 in 2024-03-07T03:47:53.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:53 smithi189 bash[66969]: cluster 2024-03-07T03:47:52.204486+0000 mon.a (mon.0) 1876 : cluster 0 osdmap e1039: 8 total, 8 up, 8 in 2024-03-07T03:47:53.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:53 smithi189 bash[66969]: cluster 2024-03-07T03:47:53.113278+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e1040: 8 total, 8 up, 8 in 2024-03-07T03:47:53.614 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnaligned (133 ms) 2024-03-07T03:47:53.614 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/0.DiffIterateTryAcquireLock 2024-03-07T03:47:53.614 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:47:54.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:54 smithi145 bash[79878]: cluster 2024-03-07T03:47:53.636066+0000 mgr.y (mgr.34107) 1081 : cluster 0 pgmap v1746: 105 pgs: 105 active+clean; 517 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 10 MiB/s wr, 392 op/s 2024-03-07T03:47:54.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:54 smithi145 bash[82329]: cluster 2024-03-07T03:47:53.636066+0000 mgr.y (mgr.34107) 1081 : cluster 0 pgmap v1746: 105 pgs: 105 active+clean; 517 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 10 MiB/s wr, 392 op/s 2024-03-07T03:47:54.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:54 smithi189 bash[66969]: cluster 2024-03-07T03:47:53.636066+0000 mgr.y (mgr.34107) 1081 : cluster 0 pgmap v1746: 105 pgs: 105 active+clean; 517 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 10 MiB/s wr, 392 op/s 2024-03-07T03:47:57.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:56 smithi145 bash[79878]: cluster 2024-03-07T03:47:55.636881+0000 mgr.y (mgr.34107) 1082 : cluster 0 pgmap v1747: 105 pgs: 105 active+clean; 529 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 9.7 MiB/s wr, 224 op/s 2024-03-07T03:47:57.067 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:56 smithi145 bash[82329]: cluster 2024-03-07T03:47:55.636881+0000 mgr.y (mgr.34107) 1082 : cluster 0 pgmap v1747: 105 pgs: 105 active+clean; 529 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 9.7 MiB/s wr, 224 op/s 2024-03-07T03:47:57.087 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:56 smithi189 bash[66969]: cluster 2024-03-07T03:47:55.636881+0000 mgr.y (mgr.34107) 1082 : cluster 0 pgmap v1747: 105 pgs: 105 active+clean; 529 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 9.7 MiB/s wr, 224 op/s 2024-03-07T03:47:58.714 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/0.DiffIterateTryAcquireLock (5099 ms) 2024-03-07T03:47:58.714 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStriping 2024-03-07T03:47:58.714 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_librbd.cc:8289: Skipped 2024-03-07T03:47:58.714 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:47:58.714 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] DiffIterateTest/0.DiffIterateStriping (1 ms) 2024-03-07T03:47:59.123 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 13 tests from DiffIterateTest/0 (63578 ms total) 2024-03-07T03:47:59.123 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:47:59.123 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 13 tests from DiffIterateTest/1, where TypeParam = DiffIterateParams 2024-03-07T03:47:59.178 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/1.DiffIterate 2024-03-07T03:47:59.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:59 smithi145 bash[79878]: cluster 2024-03-07T03:47:57.637920+0000 mgr.y (mgr.34107) 1083 : cluster 0 pgmap v1748: 105 pgs: 105 active+clean; 563 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 8.1 MiB/s wr, 235 op/s 2024-03-07T03:47:59.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:47:59 smithi145 bash[79878]: cluster 2024-03-07T03:47:58.115934+0000 mon.a (mon.0) 1878 : cluster 0 osdmap e1041: 8 total, 8 up, 8 in 2024-03-07T03:47:59.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:59 smithi145 bash[82329]: cluster 2024-03-07T03:47:57.637920+0000 mgr.y (mgr.34107) 1083 : cluster 0 pgmap v1748: 105 pgs: 105 active+clean; 563 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 8.1 MiB/s wr, 235 op/s 2024-03-07T03:47:59.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:47:59 smithi145 bash[82329]: cluster 2024-03-07T03:47:58.115934+0000 mon.a (mon.0) 1878 : cluster 0 osdmap e1041: 8 total, 8 up, 8 in 2024-03-07T03:47:59.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:59 smithi189 bash[66969]: cluster 2024-03-07T03:47:57.637920+0000 mgr.y (mgr.34107) 1083 : cluster 0 pgmap v1748: 105 pgs: 105 active+clean; 563 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 8.1 MiB/s wr, 235 op/s 2024-03-07T03:47:59.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:47:59 smithi189 bash[66969]: cluster 2024-03-07T03:47:58.115934+0000 mon.a (mon.0) 1878 : cluster 0 osdmap e1041: 8 total, 8 up, 8 in 2024-03-07T03:48:00.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:00 smithi145 bash[79878]: cluster 2024-03-07T03:47:59.117501+0000 mon.a (mon.0) 1879 : cluster 0 osdmap e1042: 8 total, 8 up, 8 in 2024-03-07T03:48:00.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:00 smithi145 bash[82329]: cluster 2024-03-07T03:47:59.117501+0000 mon.a (mon.0) 1879 : cluster 0 osdmap e1042: 8 total, 8 up, 8 in 2024-03-07T03:48:00.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:00 smithi189 bash[66969]: cluster 2024-03-07T03:47:59.117501+0000 mon.a (mon.0) 1879 : cluster 0 osdmap e1042: 8 total, 8 up, 8 in 2024-03-07T03:48:01.162 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:48:01.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:01 smithi145 bash[79878]: cluster 2024-03-07T03:47:59.638465+0000 mgr.y (mgr.34107) 1084 : cluster 0 pgmap v1751: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:48:01.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:01 smithi145 bash[79878]: cluster 2024-03-07T03:48:00.146911+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e1043: 8 total, 8 up, 8 in 2024-03-07T03:48:01.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:01 smithi145 bash[79878]: audit 2024-03-07T03:48:00.150760+0000 mon.a (mon.0) 1881 : audit 1 from='client.? 172.21.15.145:0/3859713129' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-45","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:48:01.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:01 smithi145 bash[82329]: cluster 2024-03-07T03:47:59.638465+0000 mgr.y (mgr.34107) 1084 : cluster 0 pgmap v1751: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:48:01.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:01 smithi145 bash[82329]: cluster 2024-03-07T03:48:00.146911+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e1043: 8 total, 8 up, 8 in 2024-03-07T03:48:01.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:01 smithi145 bash[82329]: audit 2024-03-07T03:48:00.150760+0000 mon.a (mon.0) 1881 : audit 1 from='client.? 172.21.15.145:0/3859713129' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-45","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:48:01.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:01 smithi189 bash[66969]: cluster 2024-03-07T03:47:59.638465+0000 mgr.y (mgr.34107) 1084 : cluster 0 pgmap v1751: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:48:01.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:01 smithi189 bash[66969]: cluster 2024-03-07T03:48:00.146911+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e1043: 8 total, 8 up, 8 in 2024-03-07T03:48:01.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:01 smithi189 bash[66969]: audit 2024-03-07T03:48:00.150760+0000 mon.a (mon.0) 1881 : audit 1 from='client.? 172.21.15.145:0/3859713129' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-45","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:48:02.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:02 smithi145 bash[79878]: audit 2024-03-07T03:48:01.148349+0000 mon.a (mon.0) 1882 : audit 1 from='client.? 172.21.15.145:0/3859713129' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-45","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:48:02.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:02 smithi145 bash[79878]: cluster 2024-03-07T03:48:01.154782+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e1044: 8 total, 8 up, 8 in 2024-03-07T03:48:02.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:02 smithi145 bash[79878]: cluster 2024-03-07T03:48:01.639155+0000 mgr.y (mgr.34107) 1085 : cluster 0 pgmap v1754: 105 pgs: 8 creating+peering, 19 unknown, 78 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:48:02.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:02 smithi145 bash[82329]: audit 2024-03-07T03:48:01.148349+0000 mon.a (mon.0) 1882 : audit 1 from='client.? 172.21.15.145:0/3859713129' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-45","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:48:02.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:02 smithi145 bash[82329]: cluster 2024-03-07T03:48:01.154782+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e1044: 8 total, 8 up, 8 in 2024-03-07T03:48:02.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:02 smithi145 bash[82329]: cluster 2024-03-07T03:48:01.639155+0000 mgr.y (mgr.34107) 1085 : cluster 0 pgmap v1754: 105 pgs: 8 creating+peering, 19 unknown, 78 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:48:02.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:02 smithi189 bash[66969]: audit 2024-03-07T03:48:01.148349+0000 mon.a (mon.0) 1882 : audit 1 from='client.? 172.21.15.145:0/3859713129' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi145-68337-45","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:48:02.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:02 smithi189 bash[66969]: cluster 2024-03-07T03:48:01.154782+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e1044: 8 total, 8 up, 8 in 2024-03-07T03:48:02.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:02 smithi189 bash[66969]: cluster 2024-03-07T03:48:01.639155+0000 mgr.y (mgr.34107) 1085 : cluster 0 pgmap v1754: 105 pgs: 8 creating+peering, 19 unknown, 78 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:48:02.588 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:48:02 smithi189 bash[59814]: ts=2024-03-07T03:48:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:48:03.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:48:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:48:02] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.43.0" 2024-03-07T03:48:03.260 INFO:tasks.workunit.client.0.smithi145.stdout: wrote [3743422~40964,3982662~75344,6511422~90838,7234498~35245,9604145~57318,12434259~17169,12495812~75569,14116490~30791,20171590~18170,20695816~72834] 2024-03-07T03:48:03.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:03 smithi145 bash[79878]: cluster 2024-03-07T03:48:02.163874+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e1045: 8 total, 8 up, 8 in 2024-03-07T03:48:03.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:03 smithi145 bash[79878]: audit 2024-03-07T03:48:02.681546+0000 mon.a (mon.0) 1885 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:03.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:03 smithi145 bash[82329]: cluster 2024-03-07T03:48:02.163874+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e1045: 8 total, 8 up, 8 in 2024-03-07T03:48:03.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:03 smithi145 bash[82329]: audit 2024-03-07T03:48:02.681546+0000 mon.a (mon.0) 1885 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:03.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:03 smithi189 bash[66969]: cluster 2024-03-07T03:48:02.163874+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e1045: 8 total, 8 up, 8 in 2024-03-07T03:48:03.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:03 smithi189 bash[66969]: audit 2024-03-07T03:48:02.681546+0000 mon.a (mon.0) 1885 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:04.227 INFO:tasks.workunit.client.0.smithi145.stdout: wrote [0~8388608,12582912~8388608] 2024-03-07T03:48:04.243 INFO:tasks.workunit.client.0.smithi145.stdout: diff was [0~8388608,12582912~8388608] 2024-03-07T03:48:04.251 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/1.DiffIterate (5073 ms) 2024-03-07T03:48:04.251 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministic 2024-03-07T03:48:04.252 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:48:04.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:04 smithi145 bash[79878]: cluster 2024-03-07T03:48:03.182855+0000 mon.a (mon.0) 1886 : cluster 0 osdmap e1046: 8 total, 8 up, 8 in 2024-03-07T03:48:04.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:04 smithi145 bash[79878]: cluster 2024-03-07T03:48:03.640055+0000 mgr.y (mgr.34107) 1086 : cluster 0 pgmap v1757: 105 pgs: 8 creating+peering, 5 unknown, 92 active+clean; 3.5 MiB data, 653 MiB used, 715 GiB / 715 GiB avail; 5.5 KiB/s rd, 31 KiB/s wr, 7 op/s 2024-03-07T03:48:04.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:04 smithi145 bash[82329]: cluster 2024-03-07T03:48:03.182855+0000 mon.a (mon.0) 1886 : cluster 0 osdmap e1046: 8 total, 8 up, 8 in 2024-03-07T03:48:04.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:04 smithi145 bash[82329]: cluster 2024-03-07T03:48:03.640055+0000 mgr.y (mgr.34107) 1086 : cluster 0 pgmap v1757: 105 pgs: 8 creating+peering, 5 unknown, 92 active+clean; 3.5 MiB data, 653 MiB used, 715 GiB / 715 GiB avail; 5.5 KiB/s rd, 31 KiB/s wr, 7 op/s 2024-03-07T03:48:04.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:04 smithi189 bash[66969]: cluster 2024-03-07T03:48:03.182855+0000 mon.a (mon.0) 1886 : cluster 0 osdmap e1046: 8 total, 8 up, 8 in 2024-03-07T03:48:04.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:04 smithi189 bash[66969]: cluster 2024-03-07T03:48:03.640055+0000 mgr.y (mgr.34107) 1086 : cluster 0 pgmap v1757: 105 pgs: 8 creating+peering, 5 unknown, 92 active+clean; 3.5 MiB data, 653 MiB used, 715 GiB / 715 GiB avail; 5.5 KiB/s rd, 31 KiB/s wr, 7 op/s 2024-03-07T03:48:05.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:05 smithi145 bash[79878]: cluster 2024-03-07T03:48:04.193450+0000 mon.a (mon.0) 1887 : cluster 0 osdmap e1047: 8 total, 8 up, 8 in 2024-03-07T03:48:05.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:05 smithi145 bash[82329]: cluster 2024-03-07T03:48:04.193450+0000 mon.a (mon.0) 1887 : cluster 0 osdmap e1047: 8 total, 8 up, 8 in 2024-03-07T03:48:05.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:05 smithi189 bash[66969]: cluster 2024-03-07T03:48:04.193450+0000 mon.a (mon.0) 1887 : cluster 0 osdmap e1047: 8 total, 8 up, 8 in 2024-03-07T03:48:06.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:06 smithi145 bash[79878]: cluster 2024-03-07T03:48:05.191893+0000 mon.a (mon.0) 1888 : cluster 0 osdmap e1048: 8 total, 8 up, 8 in 2024-03-07T03:48:06.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:06 smithi145 bash[79878]: cluster 2024-03-07T03:48:05.640766+0000 mgr.y (mgr.34107) 1087 : cluster 0 pgmap v1760: 105 pgs: 8 creating+peering, 97 active+clean; 11 MiB data, 654 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 104 KiB/s wr, 11 op/s 2024-03-07T03:48:06.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:06 smithi145 bash[82329]: cluster 2024-03-07T03:48:05.191893+0000 mon.a (mon.0) 1888 : cluster 0 osdmap e1048: 8 total, 8 up, 8 in 2024-03-07T03:48:06.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:06 smithi145 bash[82329]: cluster 2024-03-07T03:48:05.640766+0000 mgr.y (mgr.34107) 1087 : cluster 0 pgmap v1760: 105 pgs: 8 creating+peering, 97 active+clean; 11 MiB data, 654 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 104 KiB/s wr, 11 op/s 2024-03-07T03:48:06.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:06 smithi189 bash[66969]: cluster 2024-03-07T03:48:05.191893+0000 mon.a (mon.0) 1888 : cluster 0 osdmap e1048: 8 total, 8 up, 8 in 2024-03-07T03:48:06.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:06 smithi189 bash[66969]: cluster 2024-03-07T03:48:05.640766+0000 mgr.y (mgr.34107) 1087 : cluster 0 pgmap v1760: 105 pgs: 8 creating+peering, 97 active+clean; 11 MiB data, 654 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 104 KiB/s wr, 11 op/s 2024-03-07T03:48:07.280 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministic (3029 ms) 2024-03-07T03:48:07.281 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicPP 2024-03-07T03:48:07.281 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:48:07.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:07 smithi145 bash[79878]: cluster 2024-03-07T03:48:06.213811+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e1049: 8 total, 8 up, 8 in 2024-03-07T03:48:07.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:07 smithi145 bash[82329]: cluster 2024-03-07T03:48:06.213811+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e1049: 8 total, 8 up, 8 in 2024-03-07T03:48:07.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:07 smithi189 bash[66969]: cluster 2024-03-07T03:48:06.213811+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e1049: 8 total, 8 up, 8 in 2024-03-07T03:48:08.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:08 smithi145 bash[79878]: cluster 2024-03-07T03:48:07.213931+0000 mon.a (mon.0) 1890 : cluster 0 osdmap e1050: 8 total, 8 up, 8 in 2024-03-07T03:48:08.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:08 smithi145 bash[79878]: cluster 2024-03-07T03:48:07.641484+0000 mgr.y (mgr.34107) 1088 : cluster 0 pgmap v1763: 105 pgs: 105 active+clean; 19 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 231 KiB/s wr, 36 op/s 2024-03-07T03:48:08.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:08 smithi145 bash[82329]: cluster 2024-03-07T03:48:07.213931+0000 mon.a (mon.0) 1890 : cluster 0 osdmap e1050: 8 total, 8 up, 8 in 2024-03-07T03:48:08.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:08 smithi145 bash[82329]: cluster 2024-03-07T03:48:07.641484+0000 mgr.y (mgr.34107) 1088 : cluster 0 pgmap v1763: 105 pgs: 105 active+clean; 19 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 231 KiB/s wr, 36 op/s 2024-03-07T03:48:08.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:08 smithi189 bash[66969]: cluster 2024-03-07T03:48:07.213931+0000 mon.a (mon.0) 1890 : cluster 0 osdmap e1050: 8 total, 8 up, 8 in 2024-03-07T03:48:08.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:08 smithi189 bash[66969]: cluster 2024-03-07T03:48:07.641484+0000 mgr.y (mgr.34107) 1088 : cluster 0 pgmap v1763: 105 pgs: 105 active+clean; 19 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 231 KiB/s wr, 36 op/s 2024-03-07T03:48:09.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:09 smithi145 bash[79878]: cluster 2024-03-07T03:48:08.238670+0000 mon.a (mon.0) 1891 : cluster 0 osdmap e1051: 8 total, 8 up, 8 in 2024-03-07T03:48:09.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:09 smithi145 bash[82329]: cluster 2024-03-07T03:48:08.238670+0000 mon.a (mon.0) 1891 : cluster 0 osdmap e1051: 8 total, 8 up, 8 in 2024-03-07T03:48:09.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:09 smithi189 bash[66969]: cluster 2024-03-07T03:48:08.238670+0000 mon.a (mon.0) 1891 : cluster 0 osdmap e1051: 8 total, 8 up, 8 in 2024-03-07T03:48:10.317 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicPP (3037 ms) 2024-03-07T03:48:10.318 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDiscard 2024-03-07T03:48:10.318 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:48:10.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:10 smithi145 bash[79878]: cluster 2024-03-07T03:48:09.249244+0000 mon.a (mon.0) 1892 : cluster 0 osdmap e1052: 8 total, 8 up, 8 in 2024-03-07T03:48:10.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:10 smithi145 bash[79878]: cluster 2024-03-07T03:48:09.642239+0000 mgr.y (mgr.34107) 1089 : cluster 0 pgmap v1766: 105 pgs: 105 active+clean; 19 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 182 KiB/s wr, 102 op/s 2024-03-07T03:48:10.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:10 smithi145 bash[82329]: cluster 2024-03-07T03:48:09.249244+0000 mon.a (mon.0) 1892 : cluster 0 osdmap e1052: 8 total, 8 up, 8 in 2024-03-07T03:48:10.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:10 smithi145 bash[82329]: cluster 2024-03-07T03:48:09.642239+0000 mgr.y (mgr.34107) 1089 : cluster 0 pgmap v1766: 105 pgs: 105 active+clean; 19 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 182 KiB/s wr, 102 op/s 2024-03-07T03:48:10.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:10 smithi189 bash[66969]: cluster 2024-03-07T03:48:09.249244+0000 mon.a (mon.0) 1892 : cluster 0 osdmap e1052: 8 total, 8 up, 8 in 2024-03-07T03:48:10.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:10 smithi189 bash[66969]: cluster 2024-03-07T03:48:09.642239+0000 mgr.y (mgr.34107) 1089 : cluster 0 pgmap v1766: 105 pgs: 105 active+clean; 19 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 182 KiB/s wr, 102 op/s 2024-03-07T03:48:11.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:11 smithi145 bash[79878]: cluster 2024-03-07T03:48:10.260362+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e1053: 8 total, 8 up, 8 in 2024-03-07T03:48:11.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:11 smithi145 bash[82329]: cluster 2024-03-07T03:48:10.260362+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e1053: 8 total, 8 up, 8 in 2024-03-07T03:48:11.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:11 smithi189 bash[66969]: cluster 2024-03-07T03:48:10.260362+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e1053: 8 total, 8 up, 8 in 2024-03-07T03:48:12.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:12 smithi145 bash[79878]: cluster 2024-03-07T03:48:11.261013+0000 mon.a (mon.0) 1894 : cluster 0 osdmap e1054: 8 total, 8 up, 8 in 2024-03-07T03:48:12.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:12 smithi145 bash[79878]: cluster 2024-03-07T03:48:11.643008+0000 mgr.y (mgr.34107) 1090 : cluster 0 pgmap v1769: 105 pgs: 105 active+clean; 19 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 32 KiB/s wr, 110 op/s 2024-03-07T03:48:12.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:12 smithi145 bash[82329]: cluster 2024-03-07T03:48:11.261013+0000 mon.a (mon.0) 1894 : cluster 0 osdmap e1054: 8 total, 8 up, 8 in 2024-03-07T03:48:12.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:12 smithi145 bash[82329]: cluster 2024-03-07T03:48:11.643008+0000 mgr.y (mgr.34107) 1090 : cluster 0 pgmap v1769: 105 pgs: 105 active+clean; 19 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 32 KiB/s wr, 110 op/s 2024-03-07T03:48:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:12 smithi189 bash[66969]: cluster 2024-03-07T03:48:11.261013+0000 mon.a (mon.0) 1894 : cluster 0 osdmap e1054: 8 total, 8 up, 8 in 2024-03-07T03:48:12.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:12 smithi189 bash[66969]: cluster 2024-03-07T03:48:11.643008+0000 mgr.y (mgr.34107) 1090 : cluster 0 pgmap v1769: 105 pgs: 105 active+clean; 19 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 32 KiB/s wr, 110 op/s 2024-03-07T03:48:12.588 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:48:12 smithi189 bash[59814]: ts=2024-03-07T03:48:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:48:13.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:48:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:48:12] "GET /metrics HTTP/1.1" 200 36520 "" "Prometheus/2.43.0" 2024-03-07T03:48:13.197 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/1.DiffIterateDiscard (2880 ms) 2024-03-07T03:48:13.198 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStress 2024-03-07T03:48:13.198 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:48:13.292 INFO:tasks.workunit.client.0.smithi145.stdout: i=0 exists [100425120~2627541,150130395~973902,176801972~4609952,190455397~5989202,280605843~1211087,293569576~2969256,314765284~4999381,353993373~977967,359190305~734630,360839348~2916323] wrote [100425120~2627541,150130395~973902,176801972~4609952,190455397~5989202,280605843~1211087,293569576~2969256,314765284~4999381,353993373~977967,359190305~734630,360839348~2916323] 2024-03-07T03:48:13.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:13 smithi145 bash[82329]: cluster 2024-03-07T03:48:12.281741+0000 mon.a (mon.0) 1895 : cluster 0 osdmap e1055: 8 total, 8 up, 8 in 2024-03-07T03:48:13.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:13 smithi145 bash[82329]: cluster 2024-03-07T03:48:13.130131+0000 mon.a (mon.0) 1896 : cluster 0 osdmap e1056: 8 total, 8 up, 8 in 2024-03-07T03:48:13.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:13 smithi145 bash[79878]: cluster 2024-03-07T03:48:12.281741+0000 mon.a (mon.0) 1895 : cluster 0 osdmap e1055: 8 total, 8 up, 8 in 2024-03-07T03:48:13.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:13 smithi145 bash[79878]: cluster 2024-03-07T03:48:13.130131+0000 mon.a (mon.0) 1896 : cluster 0 osdmap e1056: 8 total, 8 up, 8 in 2024-03-07T03:48:13.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:13 smithi189 bash[66969]: cluster 2024-03-07T03:48:12.281741+0000 mon.a (mon.0) 1895 : cluster 0 osdmap e1055: 8 total, 8 up, 8 in 2024-03-07T03:48:13.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:13 smithi189 bash[66969]: cluster 2024-03-07T03:48:13.130131+0000 mon.a (mon.0) 1896 : cluster 0 osdmap e1056: 8 total, 8 up, 8 in 2024-03-07T03:48:14.421 INFO:tasks.workunit.client.0.smithi145.stdout: i=1 exists [54498805~6736840,63228525~5612602,80084133~2919434,100425120~2627541,109528723~10143143,150130395~973902,153190507~4702460,176801972~4609952,190455397~5989202,201768236~2275756,280605843~1211087,293569576~2969256,311337650~3081171,314765284~4999381,329129674~1648640,353993373~977967,359190305~734630,360839348~2916323,391495525~6876067] wrote [54498805~6736840,63228525~5612602,80084133~2919434,109528723~10143143,153190507~4702460,201768236~2275756,311337650~3081171,329129674~1648640,391495525~6876067] 2024-03-07T03:48:14.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:14 smithi145 bash[79878]: cluster 2024-03-07T03:48:13.643990+0000 mgr.y (mgr.34107) 1091 : cluster 0 pgmap v1772: 105 pgs: 105 active+clean; 35 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 2.9 MiB/s wr, 106 op/s 2024-03-07T03:48:14.567 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:14 smithi145 bash[82329]: cluster 2024-03-07T03:48:13.643990+0000 mgr.y (mgr.34107) 1091 : cluster 0 pgmap v1772: 105 pgs: 105 active+clean; 35 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 2.9 MiB/s wr, 106 op/s 2024-03-07T03:48:14.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:14 smithi189 bash[66969]: cluster 2024-03-07T03:48:13.643990+0000 mgr.y (mgr.34107) 1091 : cluster 0 pgmap v1772: 105 pgs: 105 active+clean; 35 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 2.9 MiB/s wr, 106 op/s 2024-03-07T03:48:15.362 INFO:tasks.workunit.client.0.smithi145.stdout: i=2 exists [15599862~2733510,54498805~6736840,63228525~5612602,71685711~897206,80084133~2919434,98873350~4179311,109528723~10143143,124915198~306230,150130395~973902,153190507~4702460,176801972~4609952,190455397~8351271,201768236~2275756,226533871~5164141,280605843~1211087,293569576~2969256,298224995~1224276,311337650~3081171,314765284~4999381,329129674~1648640,338357992~1253983,353993373~977967,359190305~734630,360839348~2916323,384413183~1411994,391495525~6876067] wrote [15599862~2733510,71685711~897206,98873350~1938306,124915198~306230,194368056~4438612,226533871~5164141,298224995~1224276,338357992~1253983,384413183~1411994] 2024-03-07T03:48:15.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:15 smithi145 bash[79878]: cluster 2024-03-07T03:48:14.292202+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e1057: 8 total, 8 up, 8 in 2024-03-07T03:48:15.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:15 smithi145 bash[79878]: audit 2024-03-07T03:48:14.781906+0000 mon.a (mon.0) 1898 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:15.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:15 smithi145 bash[82329]: cluster 2024-03-07T03:48:14.292202+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e1057: 8 total, 8 up, 8 in 2024-03-07T03:48:15.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:15 smithi145 bash[82329]: audit 2024-03-07T03:48:14.781906+0000 mon.a (mon.0) 1898 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:15.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:15 smithi189 bash[66969]: cluster 2024-03-07T03:48:14.292202+0000 mon.a (mon.0) 1897 : cluster 0 osdmap e1057: 8 total, 8 up, 8 in 2024-03-07T03:48:15.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:15 smithi189 bash[66969]: audit 2024-03-07T03:48:14.781906+0000 mon.a (mon.0) 1898 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:16.396 INFO:tasks.workunit.client.0.smithi145.stdout: i=3 exists [15599862~2733510,54498805~6736840,63228525~5612602,71685711~897206,80084133~2919434,98873350~4179311,109528723~10143143,122822535~4063321,139055012~4345938,150130395~973902,153190507~4702460,176801972~4609952,190455397~8351271,200362721~5685003,206993521~9910395,226533871~5164141,280605843~1211087,293569576~2969256,298224995~1224276,311337650~3081171,314765284~4999381,329129674~1648640,338357992~1253983,341664823~1419274,343892801~3501528,353993373~977967,359190305~734630,360839348~2916323,383911446~225855,384413183~1411994,391495525~6876067] wrote [122822535~4063321,139055012~4345938,200362721~5685003,206993521~9910395,313289312~371885,341664823~1419274,343892801~3501528,383911446~225855,391955512~6019232] 2024-03-07T03:48:16.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:16 smithi145 bash[79878]: cluster 2024-03-07T03:48:15.304208+0000 mon.a (mon.0) 1899 : cluster 0 osdmap e1058: 8 total, 8 up, 8 in 2024-03-07T03:48:16.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:16 smithi145 bash[79878]: cluster 2024-03-07T03:48:15.644688+0000 mgr.y (mgr.34107) 1092 : cluster 0 pgmap v1775: 105 pgs: 105 active+clean; 49 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 5.2 MiB/s wr, 73 op/s 2024-03-07T03:48:16.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:16 smithi145 bash[82329]: cluster 2024-03-07T03:48:15.304208+0000 mon.a (mon.0) 1899 : cluster 0 osdmap e1058: 8 total, 8 up, 8 in 2024-03-07T03:48:16.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:16 smithi145 bash[82329]: cluster 2024-03-07T03:48:15.644688+0000 mgr.y (mgr.34107) 1092 : cluster 0 pgmap v1775: 105 pgs: 105 active+clean; 49 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 5.2 MiB/s wr, 73 op/s 2024-03-07T03:48:16.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:16 smithi189 bash[66969]: cluster 2024-03-07T03:48:15.304208+0000 mon.a (mon.0) 1899 : cluster 0 osdmap e1058: 8 total, 8 up, 8 in 2024-03-07T03:48:16.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:16 smithi189 bash[66969]: cluster 2024-03-07T03:48:15.644688+0000 mgr.y (mgr.34107) 1092 : cluster 0 pgmap v1775: 105 pgs: 105 active+clean; 49 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 5.2 MiB/s wr, 73 op/s 2024-03-07T03:48:17.429 INFO:tasks.workunit.client.0.smithi145.stdout: i=4 exists [14773258~3686073,43747609~6039631,54498805~6736840,63228525~5612602,71685711~897206,76401250~6602317,84113477~186314,98873350~4179311,109528723~10143143,122822535~4063321,139055012~4345938,150130395~973902,153190507~6086236,176801972~4609952,190455397~8351271,200362721~5685003,206993521~9910395,222294994~4148591,226533871~5164141,254885544~5460772,280605843~1211087,291638389~7866285,311337650~3081171,314765284~4999381,329129674~1648640,338357992~1253983,341664823~1419274,343892801~3501528,353993373~977967,359190305~734630,360839348~8087320,383911446~225855,384413183~1411994,391495525~6876067,404816244~252009] wrote [14773258~3686073,43747609~6039631,76401250~4705298,84113477~186314,156134217~3142526,222294994~4148591,254885544~5460772,291638389~7866285,361944311~6982357,404816244~252009] 2024-03-07T03:48:17.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:17 smithi145 bash[79878]: cluster 2024-03-07T03:48:16.311322+0000 mon.a (mon.0) 1900 : cluster 0 osdmap e1059: 8 total, 8 up, 8 in 2024-03-07T03:48:17.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:17 smithi145 bash[82329]: cluster 2024-03-07T03:48:16.311322+0000 mon.a (mon.0) 1900 : cluster 0 osdmap e1059: 8 total, 8 up, 8 in 2024-03-07T03:48:17.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:17 smithi189 bash[66969]: cluster 2024-03-07T03:48:16.311322+0000 mon.a (mon.0) 1900 : cluster 0 osdmap e1059: 8 total, 8 up, 8 in 2024-03-07T03:48:18.485 INFO:tasks.workunit.client.0.smithi145.stdout: i=5 exists [5879476~8041759,14773258~3686073,39679362~9134,43747609~6039631,54498805~6736840,63228525~5612602,71685711~897206,76401250~6602317,84113477~186314,98873350~4179311,109528723~10143143,122822535~4063321,139055012~4345938,150130395~973902,153190507~6086236,176801972~4609952,183718918~3567549,190455397~8351271,200362721~16541195,222294994~4148591,226533871~5164141,254885544~5460772,280605843~1211087,283471402~16033272,311337650~3081171,314765284~4999381,329129674~1648640,338357992~1253983,341664823~1419274,343892801~3501528,353993373~977967,359190305~734630,360839348~8087320,370314467~1743386,383911446~225855,384413183~1411994,386137161~12234431,402764712~1521812,404816244~252009] wrote [5879476~8041759,39679362~9134,183718918~3567549,202883084~4471232,283471402~11915729,370314467~1743386,386137161~6800932,402764712~1521812] 2024-03-07T03:48:18.587 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:18 smithi189 bash[66969]: cluster 2024-03-07T03:48:17.313286+0000 mon.a (mon.0) 1901 : cluster 0 osdmap e1060: 8 total, 8 up, 8 in 2024-03-07T03:48:18.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:18 smithi189 bash[66969]: cluster 2024-03-07T03:48:17.645437+0000 mgr.y (mgr.34107) 1093 : cluster 0 pgmap v1778: 105 pgs: 105 active+clean; 125 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 15 MiB/s wr, 167 op/s 2024-03-07T03:48:18.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:18 smithi189 bash[66969]: audit 2024-03-07T03:48:17.681915+0000 mon.a (mon.0) 1902 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:18.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:18 smithi145 bash[79878]: cluster 2024-03-07T03:48:17.313286+0000 mon.a (mon.0) 1901 : cluster 0 osdmap e1060: 8 total, 8 up, 8 in 2024-03-07T03:48:18.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:18 smithi145 bash[79878]: cluster 2024-03-07T03:48:17.645437+0000 mgr.y (mgr.34107) 1093 : cluster 0 pgmap v1778: 105 pgs: 105 active+clean; 125 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 15 MiB/s wr, 167 op/s 2024-03-07T03:48:18.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:18 smithi145 bash[79878]: audit 2024-03-07T03:48:17.681915+0000 mon.a (mon.0) 1902 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:18.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:18 smithi145 bash[82329]: cluster 2024-03-07T03:48:17.313286+0000 mon.a (mon.0) 1901 : cluster 0 osdmap e1060: 8 total, 8 up, 8 in 2024-03-07T03:48:18.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:18 smithi145 bash[82329]: cluster 2024-03-07T03:48:17.645437+0000 mgr.y (mgr.34107) 1093 : cluster 0 pgmap v1778: 105 pgs: 105 active+clean; 125 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 15 MiB/s wr, 167 op/s 2024-03-07T03:48:18.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:18 smithi145 bash[82329]: audit 2024-03-07T03:48:17.681915+0000 mon.a (mon.0) 1902 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:19.434 INFO:tasks.workunit.client.0.smithi145.stdout: i=6 exists [5879476~8041759,14773258~3686073,29619687~4025413,39679362~9134,43747609~6039631,54498805~6736840,63228525~5612602,71685711~897206,76401250~6602317,84113477~186314,98873350~4179311,107510194~12161672,122822535~4063321,139055012~4345938,149370843~1733454,153190507~6086236,176801972~4609952,181688138~981042,183718918~3630549,190455397~8351271,200362721~16541195,222294994~4148591,226533871~5164141,236699682~10754565,254885544~5460772,270726847~3085359,280605843~1211087,283471402~16033272,311337650~3081171,314765284~5824054,329129674~1648640,338357992~1253983,341664823~1419274,343892801~3501528,353993373~977967,359190305~734630,360839348~8087320,370314467~1743386,383911446~225855,384413183~1411994,386137161~12234431,402764712~1521812,404816244~252009] wrote [29619687~4025413,107510194~3534613,149370843~1488376,181688138~981042,185568659~1780808,211583750~360813,236699682~10754565,270726847~3085359,316767257~3822081] 2024-03-07T03:48:19.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:19 smithi189 bash[66969]: cluster 2024-03-07T03:48:18.332098+0000 mon.a (mon.0) 1903 : cluster 0 osdmap e1061: 8 total, 8 up, 8 in 2024-03-07T03:48:19.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:19 smithi145 bash[79878]: cluster 2024-03-07T03:48:18.332098+0000 mon.a (mon.0) 1903 : cluster 0 osdmap e1061: 8 total, 8 up, 8 in 2024-03-07T03:48:19.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:19 smithi145 bash[82329]: cluster 2024-03-07T03:48:18.332098+0000 mon.a (mon.0) 1903 : cluster 0 osdmap e1061: 8 total, 8 up, 8 in 2024-03-07T03:48:20.466 INFO:tasks.workunit.client.0.smithi145.stdout: i=7 exists [5879476~8041759,14773258~3686073,29619687~4025413,35291787~2972551,39679362~9134,43747609~6039631,54498805~6736840,61994144~6846983,70507758~12495809,84113477~186314,98873350~4179311,107510194~12161672,122822535~4063321,139055012~4345938,149370843~1733454,153190507~6086236,176801972~4609952,181688138~981042,183718918~3630549,190455397~8351271,200362721~18823706,219751002~6692583,226533871~5164141,234880366~12573881,254885544~5460772,270726847~3085359,280605843~1211087,283471402~16033272,311337650~3081171,314765284~5824054,329129674~1648640,338357992~1253983,341664823~1419274,343892801~3501528,349541543~7137863,359190305~734630,360839348~11869572,383911446~225855,384413183~1411994,386137161~12234431,402764712~1521812,404816244~252009] wrote [35291787~2972551,61994144~6200600,70507758~7020456,215170215~4016212,219751002~2722394,234880366~8074198,349541543~7137863,366289624~6419296] 2024-03-07T03:48:20.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:20 smithi145 bash[79878]: cluster 2024-03-07T03:48:19.333847+0000 mon.a (mon.0) 1904 : cluster 0 osdmap e1062: 8 total, 8 up, 8 in 2024-03-07T03:48:20.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:20 smithi145 bash[79878]: cluster 2024-03-07T03:48:19.646198+0000 mgr.y (mgr.34107) 1094 : cluster 0 pgmap v1781: 105 pgs: 105 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 99 KiB/s rd, 35 MiB/s wr, 171 op/s 2024-03-07T03:48:20.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:20 smithi145 bash[82329]: cluster 2024-03-07T03:48:19.333847+0000 mon.a (mon.0) 1904 : cluster 0 osdmap e1062: 8 total, 8 up, 8 in 2024-03-07T03:48:20.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:20 smithi145 bash[82329]: cluster 2024-03-07T03:48:19.646198+0000 mgr.y (mgr.34107) 1094 : cluster 0 pgmap v1781: 105 pgs: 105 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 99 KiB/s rd, 35 MiB/s wr, 171 op/s 2024-03-07T03:48:20.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:20 smithi189 bash[66969]: cluster 2024-03-07T03:48:19.333847+0000 mon.a (mon.0) 1904 : cluster 0 osdmap e1062: 8 total, 8 up, 8 in 2024-03-07T03:48:20.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:20 smithi189 bash[66969]: cluster 2024-03-07T03:48:19.646198+0000 mgr.y (mgr.34107) 1094 : cluster 0 pgmap v1781: 105 pgs: 105 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 99 KiB/s rd, 35 MiB/s wr, 171 op/s 2024-03-07T03:48:21.455 INFO:tasks.workunit.client.0.smithi145.stdout: i=8 exists [1702466~2087664,5879476~8041759,14773258~3686073,23679409~458158,29619687~4025413,35291787~2972551,39679362~9134,41750213~10564102,54498805~6736840,61994144~6846983,70507758~12495809,84113477~186314,98873350~4179311,107510194~12161672,122822535~4063321,133804353~9596597,149370843~1733454,153190507~6086236,176801972~4609952,181688138~981042,183718918~3630549,190455397~8351271,200362721~18823706,219751002~6692583,226533871~5164141,234880366~12573881,254885544~5460772,264086995~5499750,270726847~3085359,280605843~1211087,283471402~16033272,311337650~3081171,314765284~6421297,329129674~1648640,338357992~1253983,341664823~1419274,343892801~12786605,359190305~734630,360839348~11869572,383911446~225855,384413183~1411994,386137161~12234431,402764712~1521812,404816244~252009] wrote [1702466~2087664,23679409~458158,41750213~6267412,49305514~3008801,71139250~2750393,133804353~7884894,264086995~5499750,316153156~5033425,346863638~5278806] 2024-03-07T03:48:21.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:21 smithi145 bash[79878]: cluster 2024-03-07T03:48:20.354930+0000 mon.a (mon.0) 1905 : cluster 0 osdmap e1063: 8 total, 8 up, 8 in 2024-03-07T03:48:21.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:21 smithi145 bash[82329]: cluster 2024-03-07T03:48:20.354930+0000 mon.a (mon.0) 1905 : cluster 0 osdmap e1063: 8 total, 8 up, 8 in 2024-03-07T03:48:21.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:21 smithi189 bash[66969]: cluster 2024-03-07T03:48:20.354930+0000 mon.a (mon.0) 1905 : cluster 0 osdmap e1063: 8 total, 8 up, 8 in 2024-03-07T03:48:22.488 INFO:tasks.workunit.client.0.smithi145.stdout: i=9 exists [1702466~2087664,5879476~8041759,14773258~3686073,23679409~458158,27593259~6362413,35291787~2972551,39679362~9134,41750213~10564102,54498805~6736840,61715007~7126120,70507758~12495809,84113477~186314,98507705~6816926,107510194~12161672,122822535~4063321,133804353~9596597,149370843~1733454,153190507~6086236,176801972~4609952,181688138~981042,183718918~3630549,190455397~8351271,200362721~18823706,219751002~6692583,226533871~5164141,234880366~12573881,254885544~5460772,264086995~5499750,270726847~3085359,277821495~24545553,311337650~3081171,314696020~6490561,329129674~1648640,338357992~1253983,341664823~1419274,343892801~12786605,359190305~734630,360839348~11869572,383911446~225855,384413183~1411994,386137161~12234431,402764712~1521812,404816244~252009] wrote [27593259~6362413,61715007~2524972,75317132~6515186,98507705~6816926,156664336~1236149,239725108~4285322,277821495~7848368,296111271~6255777,314696020~3210964] 2024-03-07T03:48:22.588 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:48:22 smithi189 bash[59814]: ts=2024-03-07T03:48:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:48:22.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:22 smithi189 bash[66969]: cluster 2024-03-07T03:48:21.360097+0000 mon.a (mon.0) 1906 : cluster 0 osdmap e1064: 8 total, 8 up, 8 in 2024-03-07T03:48:22.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:22 smithi189 bash[66969]: cluster 2024-03-07T03:48:21.646949+0000 mgr.y (mgr.34107) 1095 : cluster 0 pgmap v1784: 105 pgs: 105 active+clean; 347 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 109 KiB/s rd, 49 MiB/s wr, 198 op/s 2024-03-07T03:48:22.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:22 smithi189 bash[66969]: audit 2024-03-07T03:48:21.984188+0000 mon.a (mon.0) 1907 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:48:22.759 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:22 smithi145 bash[82329]: cluster 2024-03-07T03:48:21.360097+0000 mon.a (mon.0) 1906 : cluster 0 osdmap e1064: 8 total, 8 up, 8 in 2024-03-07T03:48:22.759 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:22 smithi145 bash[82329]: cluster 2024-03-07T03:48:21.646949+0000 mgr.y (mgr.34107) 1095 : cluster 0 pgmap v1784: 105 pgs: 105 active+clean; 347 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 109 KiB/s rd, 49 MiB/s wr, 198 op/s 2024-03-07T03:48:22.759 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:22 smithi145 bash[82329]: audit 2024-03-07T03:48:21.984188+0000 mon.a (mon.0) 1907 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:48:22.760 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:22 smithi145 bash[79878]: cluster 2024-03-07T03:48:21.360097+0000 mon.a (mon.0) 1906 : cluster 0 osdmap e1064: 8 total, 8 up, 8 in 2024-03-07T03:48:22.760 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:22 smithi145 bash[79878]: cluster 2024-03-07T03:48:21.646949+0000 mgr.y (mgr.34107) 1095 : cluster 0 pgmap v1784: 105 pgs: 105 active+clean; 347 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 109 KiB/s rd, 49 MiB/s wr, 198 op/s 2024-03-07T03:48:22.760 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:22 smithi145 bash[79878]: audit 2024-03-07T03:48:21.984188+0000 mon.a (mon.0) 1907 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:48:23.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:48:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:48:22] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.43.0" 2024-03-07T03:48:23.478 INFO:tasks.workunit.client.0.smithi145.stdout: i=10 exists [1702466~2087664,5879476~8041759,14773258~3686073,23679409~458158,27593259~6362413,35291787~2972551,39679362~9134,41750213~10564102,54498805~6736840,61715007~7126120,70507758~12495809,83399824~3668265,98507705~6816926,105454278~14217588,122822535~4063321,133804353~9596597,149370843~9905900,176801972~4609952,181607396~5896839,190455397~8351271,200362721~18823706,219751002~6692583,226533871~5164141,234880366~12573881,254885544~5460772,264086995~5499750,270726847~3085359,277821495~24545553,311337650~3081171,314696020~6490561,329129674~1648640,338357992~1253983,341664823~1419274,343892801~12786605,359190305~734630,360839348~11869572,381947474~719903,382802835~1447468,384413183~1411994,386137161~12234431,402764712~1521812,404816244~252009] wrote [48972398~1871679,83399824~3668265,105454278~3317441,151010650~7311350,181607396~5896839,195220853~105289,237309409~7595067,349618412~2578714,381947474~719903,382802835~1447468] 2024-03-07T03:48:23.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:23 smithi145 bash[82329]: cluster 2024-03-07T03:48:22.379707+0000 mon.a (mon.0) 1908 : cluster 0 osdmap e1065: 8 total, 8 up, 8 in 2024-03-07T03:48:23.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:23 smithi145 bash[82329]: audit 2024-03-07T03:48:22.562114+0000 mon.a (mon.0) 1909 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:23.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:23 smithi145 bash[82329]: audit 2024-03-07T03:48:22.567442+0000 mon.a (mon.0) 1910 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:23.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:23 smithi145 bash[79878]: cluster 2024-03-07T03:48:22.379707+0000 mon.a (mon.0) 1908 : cluster 0 osdmap e1065: 8 total, 8 up, 8 in 2024-03-07T03:48:23.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:23 smithi145 bash[79878]: audit 2024-03-07T03:48:22.562114+0000 mon.a (mon.0) 1909 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:23.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:23 smithi145 bash[79878]: audit 2024-03-07T03:48:22.567442+0000 mon.a (mon.0) 1910 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:23.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:23 smithi189 bash[66969]: cluster 2024-03-07T03:48:22.379707+0000 mon.a (mon.0) 1908 : cluster 0 osdmap e1065: 8 total, 8 up, 8 in 2024-03-07T03:48:23.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:23 smithi189 bash[66969]: audit 2024-03-07T03:48:22.562114+0000 mon.a (mon.0) 1909 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:23.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:23 smithi189 bash[66969]: audit 2024-03-07T03:48:22.567442+0000 mon.a (mon.0) 1910 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:24.515 INFO:tasks.workunit.client.0.smithi145.stdout: i=11 exists [1702466~2087664,5879476~8041759,14637000~8390242,23679409~458158,27593259~6362413,35291787~2972551,39679362~9134,41750213~10564102,54498805~14342322,70507758~12495809,83399824~7632513,98507705~6816926,105454278~14217588,122822535~4063321,131009738~12391212,149370843~9905900,176801972~4609952,181607396~5896839,190455397~8351271,200362721~18823706,219751002~6692583,226533871~5164141,234880366~12573881,254885544~5460772,264086995~9725211,277821495~24545553,311337650~3081171,314696020~6490561,329129674~1648640,338357992~1253983,341664823~1419274,343892801~12786605,359190305~734630,360839348~11869572,381947474~719903,382802835~1447468,384413183~1411994,386137161~13200831,402764712~1521812,404816244~252009] wrote [14637000~8390242,58431609~4585123,72358610~4131135,85164868~5867469,131009738~6696255,268622523~3366212,364169645~4878541,392646486~6691506] 2024-03-07T03:48:24.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:24 smithi145 bash[79878]: cluster 2024-03-07T03:48:23.379418+0000 mon.a (mon.0) 1911 : cluster 0 osdmap e1066: 8 total, 8 up, 8 in 2024-03-07T03:48:24.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:24 smithi145 bash[79878]: cluster 2024-03-07T03:48:23.647856+0000 mgr.y (mgr.34107) 1096 : cluster 0 pgmap v1787: 105 pgs: 105 active+clean; 436 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 171 KiB/s rd, 46 MiB/s wr, 281 op/s 2024-03-07T03:48:24.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:24 smithi145 bash[82329]: cluster 2024-03-07T03:48:23.379418+0000 mon.a (mon.0) 1911 : cluster 0 osdmap e1066: 8 total, 8 up, 8 in 2024-03-07T03:48:24.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:24 smithi145 bash[82329]: cluster 2024-03-07T03:48:23.647856+0000 mgr.y (mgr.34107) 1096 : cluster 0 pgmap v1787: 105 pgs: 105 active+clean; 436 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 171 KiB/s rd, 46 MiB/s wr, 281 op/s 2024-03-07T03:48:24.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:24 smithi189 bash[66969]: cluster 2024-03-07T03:48:23.379418+0000 mon.a (mon.0) 1911 : cluster 0 osdmap e1066: 8 total, 8 up, 8 in 2024-03-07T03:48:24.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:24 smithi189 bash[66969]: cluster 2024-03-07T03:48:23.647856+0000 mgr.y (mgr.34107) 1096 : cluster 0 pgmap v1787: 105 pgs: 105 active+clean; 436 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 171 KiB/s rd, 46 MiB/s wr, 281 op/s 2024-03-07T03:48:25.565 INFO:tasks.workunit.client.0.smithi145.stdout: i=12 exists [1702466~2087664,5879476~8041759,14637000~8390242,23679409~458158,27593259~6362413,35291787~17022528,54498805~14342322,70507758~12495809,83399824~7632513,98507705~6816926,105454278~14217588,122822535~4063321,131009738~12391212,149370843~9905900,168471206~6738536,176801972~4609952,181607396~5896839,190455397~8351271,200362721~18823706,219751002~6692583,226533871~5164141,234880366~12573881,254885544~5460772,261042027~13404274,277821495~24545553,311337650~3081171,314696020~6490561,329129674~1648640,338357992~1253983,341664823~1419274,343892801~12786605,359190305~734630,360839348~11869572,381947474~719903,382802835~1447468,384413183~1411994,386137161~13200831,402764712~1521812,404816244~252009] wrote [7588751~3342486,35590519~7900341,77604397~4705544,168471206~6738536,222428338~1670030,239377019~4947965,261042027~7422211,271572471~2873830,290645449~1195834,364708695~5488289] 2024-03-07T03:48:25.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:25 smithi145 bash[79878]: cluster 2024-03-07T03:48:24.404325+0000 mon.a (mon.0) 1912 : cluster 0 osdmap e1067: 8 total, 8 up, 8 in 2024-03-07T03:48:25.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:25 smithi145 bash[82329]: cluster 2024-03-07T03:48:24.404325+0000 mon.a (mon.0) 1912 : cluster 0 osdmap e1067: 8 total, 8 up, 8 in 2024-03-07T03:48:25.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:25 smithi189 bash[66969]: cluster 2024-03-07T03:48:24.404325+0000 mon.a (mon.0) 1912 : cluster 0 osdmap e1067: 8 total, 8 up, 8 in 2024-03-07T03:48:26.568 INFO:tasks.workunit.client.0.smithi145.stdout: i=13 exists [1702466~2087664,5879476~8041759,14637000~8390242,23679409~458158,27593259~6362413,35291787~17022528,54498805~14342322,70507758~12495809,83399824~7632513,98507705~6816926,105454278~14217588,122822535~4063321,131009738~12391212,149370843~9905900,160441040~7060898,168471206~6738536,176801972~13568282,190455397~8351271,200362721~18823706,219751002~6692583,226533871~7162347,234880366~12573881,254885544~5460772,261042027~13404274,277821495~25996465,311337650~3081171,314696020~6490561,329129674~1648640,338357992~1253983,341664823~1419274,343892801~12786605,359190305~734630,360839348~11869572,381947474~719903,382802835~16535157,402764712~1521812,404497225~1405484] wrote [66126024~1308175,160441040~7060898,180002372~10367882,231474989~2221229,300683818~3134142,382871625~4732102,404497225~1405484] 2024-03-07T03:48:26.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:26 smithi145 bash[79878]: cluster 2024-03-07T03:48:25.412466+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e1068: 8 total, 8 up, 8 in 2024-03-07T03:48:26.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:26 smithi145 bash[79878]: cluster 2024-03-07T03:48:25.648548+0000 mgr.y (mgr.34107) 1097 : cluster 0 pgmap v1790: 105 pgs: 105 active+clean; 452 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 22 MiB/s wr, 97 op/s 2024-03-07T03:48:26.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:26 smithi145 bash[82329]: cluster 2024-03-07T03:48:25.412466+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e1068: 8 total, 8 up, 8 in 2024-03-07T03:48:26.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:26 smithi145 bash[82329]: cluster 2024-03-07T03:48:25.648548+0000 mgr.y (mgr.34107) 1097 : cluster 0 pgmap v1790: 105 pgs: 105 active+clean; 452 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 22 MiB/s wr, 97 op/s 2024-03-07T03:48:26.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:26 smithi189 bash[66969]: cluster 2024-03-07T03:48:25.412466+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e1068: 8 total, 8 up, 8 in 2024-03-07T03:48:26.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:26 smithi189 bash[66969]: cluster 2024-03-07T03:48:25.648548+0000 mgr.y (mgr.34107) 1097 : cluster 0 pgmap v1790: 105 pgs: 105 active+clean; 452 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 22 MiB/s wr, 97 op/s 2024-03-07T03:48:27.520 INFO:tasks.workunit.client.0.smithi145.stdout: i=14 exists [1702466~2087664,5879476~8041759,14637000~8390242,23679409~458158,27593259~6362413,35291787~17022528,54498805~14342322,70507758~12495809,83399824~7632513,98507705~6816926,105454278~14217588,122822535~20578415,149370843~9905900,160441040~7060898,168471206~6738536,176801972~13568282,190455397~8351271,200362721~18823706,219751002~6692583,226533871~7162347,234880366~12573881,253247316~1271655,254885544~5460772,261042027~13404274,277821495~25996465,311337650~3081171,314696020~6490561,329129674~6464120,338357992~1253983,341664823~1419274,343892801~12786605,359190305~13518615,376309184~1804320,381947474~719903,382802835~16535157,402764712~1521812,404497225~1405484] wrote [36584464~1652014,126069921~7266805,212099565~3567372,220487026~2532334,242634175~1212609,253247316~1271655,330334491~5259303,348375326~6558297,359295918~3360545,376309184~1804320] 2024-03-07T03:48:27.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:27 smithi145 bash[79878]: cluster 2024-03-07T03:48:26.421743+0000 mon.a (mon.0) 1914 : cluster 0 osdmap e1069: 8 total, 8 up, 8 in 2024-03-07T03:48:27.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:27 smithi145 bash[82329]: cluster 2024-03-07T03:48:26.421743+0000 mon.a (mon.0) 1914 : cluster 0 osdmap e1069: 8 total, 8 up, 8 in 2024-03-07T03:48:27.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:27 smithi189 bash[66969]: cluster 2024-03-07T03:48:26.421743+0000 mon.a (mon.0) 1914 : cluster 0 osdmap e1069: 8 total, 8 up, 8 in 2024-03-07T03:48:28.572 INFO:tasks.workunit.client.0.smithi145.stdout: i=15 exists [1702466~2087664,5879476~8041759,14637000~8390242,23679409~458158,27593259~6362413,35291787~17022528,54498805~14342322,70507758~12495809,83399824~7632513,98507705~21910168,122822535~20578415,149370843~9905900,160441040~7060898,168471206~30335462,200362721~18823706,219751002~6692583,226533871~7162347,234880366~12573881,253247316~1271655,254885544~5460772,261042027~13404274,277821495~25996465,311337650~3081171,314696020~6490561,329129674~6464120,338357992~1253983,341664823~1419274,343892801~12786605,359190305~13518615,376309184~1804320,381947474~719903,382802835~16535157,402764712~1521812,404497225~1405484] wrote [17007410~3023561,60011908~3871992,101579170~7166764,119312236~1105637,130094613~7500109,171788543~8043700,189967077~7037472,209200650~100595,281985237~5748483] 2024-03-07T03:48:28.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:28 smithi145 bash[82329]: cluster 2024-03-07T03:48:27.439967+0000 mon.a (mon.0) 1915 : cluster 0 osdmap e1070: 8 total, 8 up, 8 in 2024-03-07T03:48:28.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:28 smithi145 bash[82329]: cluster 2024-03-07T03:48:27.648966+0000 mgr.y (mgr.34107) 1098 : cluster 0 pgmap v1793: 105 pgs: 105 active+clean; 548 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 215 KiB/s rd, 26 MiB/s wr, 331 op/s 2024-03-07T03:48:28.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:28 smithi145 bash[82329]: audit 2024-03-07T03:48:28.122360+0000 mon.a (mon.0) 1916 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:28.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:28 smithi145 bash[82329]: audit 2024-03-07T03:48:28.128704+0000 mon.a (mon.0) 1917 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:28.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:28 smithi145 bash[82329]: audit 2024-03-07T03:48:28.358657+0000 mon.a (mon.0) 1918 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:28.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:28 smithi145 bash[82329]: audit 2024-03-07T03:48:28.371572+0000 mon.a (mon.0) 1919 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:28.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:28 smithi145 bash[79878]: cluster 2024-03-07T03:48:27.439967+0000 mon.a (mon.0) 1915 : cluster 0 osdmap e1070: 8 total, 8 up, 8 in 2024-03-07T03:48:28.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:28 smithi145 bash[79878]: cluster 2024-03-07T03:48:27.648966+0000 mgr.y (mgr.34107) 1098 : cluster 0 pgmap v1793: 105 pgs: 105 active+clean; 548 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 215 KiB/s rd, 26 MiB/s wr, 331 op/s 2024-03-07T03:48:28.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:28 smithi145 bash[79878]: audit 2024-03-07T03:48:28.122360+0000 mon.a (mon.0) 1916 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:28.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:28 smithi145 bash[79878]: audit 2024-03-07T03:48:28.128704+0000 mon.a (mon.0) 1917 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:28.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:28 smithi145 bash[79878]: audit 2024-03-07T03:48:28.358657+0000 mon.a (mon.0) 1918 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:28.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:28 smithi145 bash[79878]: audit 2024-03-07T03:48:28.371572+0000 mon.a (mon.0) 1919 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:28.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:28 smithi189 bash[66969]: cluster 2024-03-07T03:48:27.439967+0000 mon.a (mon.0) 1915 : cluster 0 osdmap e1070: 8 total, 8 up, 8 in 2024-03-07T03:48:28.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:28 smithi189 bash[66969]: cluster 2024-03-07T03:48:27.648966+0000 mgr.y (mgr.34107) 1098 : cluster 0 pgmap v1793: 105 pgs: 105 active+clean; 548 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 215 KiB/s rd, 26 MiB/s wr, 331 op/s 2024-03-07T03:48:28.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:28 smithi189 bash[66969]: audit 2024-03-07T03:48:28.122360+0000 mon.a (mon.0) 1916 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:28.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:28 smithi189 bash[66969]: audit 2024-03-07T03:48:28.128704+0000 mon.a (mon.0) 1917 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:28.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:28 smithi189 bash[66969]: audit 2024-03-07T03:48:28.358657+0000 mon.a (mon.0) 1918 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:28.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:28 smithi189 bash[66969]: audit 2024-03-07T03:48:28.371572+0000 mon.a (mon.0) 1919 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:29.572 INFO:tasks.workunit.client.0.smithi145.stdout: i=16 exists [1702466~2087664,5879476~8041759,14637000~8390242,23679409~458158,27593259~6362413,35291787~17022528,54498805~15219468,70507758~12495809,83399824~7632513,95698331~342243,98507705~21910168,122822535~20578415,149370843~9905900,160441040~7060898,168471206~31307322,200362721~18823706,219751002~6692583,226533871~7162347,234880366~12573881,253247316~7099000,261042027~13404274,277821495~25996465,311337650~3081171,314696020~6490561,329129674~6464120,338357992~1253983,341664823~1419274,343892801~34220703,381947474~719903,382802835~16535157,402764712~1521812,404497225~5260784] wrote [61660013~8058260,95698331~342243,111068302~2771565,191800723~7977805,208191004~4682761,253570008~4570677,261724543~1280864,354382966~6421881,370016619~6319914,404601247~5156762] 2024-03-07T03:48:29.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:29 smithi145 bash[79878]: cluster 2024-03-07T03:48:28.454253+0000 mon.a (mon.0) 1920 : cluster 0 osdmap e1071: 8 total, 8 up, 8 in 2024-03-07T03:48:29.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:29 smithi145 bash[79878]: audit 2024-03-07T03:48:29.000323+0000 mon.a (mon.0) 1921 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:48:29.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:29 smithi145 bash[79878]: audit 2024-03-07T03:48:29.001768+0000 mon.a (mon.0) 1922 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:48:29.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:29 smithi145 bash[79878]: audit 2024-03-07T03:48:29.011790+0000 mon.a (mon.0) 1923 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:29.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:29 smithi145 bash[82329]: cluster 2024-03-07T03:48:28.454253+0000 mon.a (mon.0) 1920 : cluster 0 osdmap e1071: 8 total, 8 up, 8 in 2024-03-07T03:48:29.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:29 smithi145 bash[82329]: audit 2024-03-07T03:48:29.000323+0000 mon.a (mon.0) 1921 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:48:29.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:29 smithi145 bash[82329]: audit 2024-03-07T03:48:29.001768+0000 mon.a (mon.0) 1922 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:48:29.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:29 smithi145 bash[82329]: audit 2024-03-07T03:48:29.011790+0000 mon.a (mon.0) 1923 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:29.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:29 smithi189 bash[66969]: cluster 2024-03-07T03:48:28.454253+0000 mon.a (mon.0) 1920 : cluster 0 osdmap e1071: 8 total, 8 up, 8 in 2024-03-07T03:48:29.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:29 smithi189 bash[66969]: audit 2024-03-07T03:48:29.000323+0000 mon.a (mon.0) 1921 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:48:29.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:29 smithi189 bash[66969]: audit 2024-03-07T03:48:29.001768+0000 mon.a (mon.0) 1922 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:48:29.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:29 smithi189 bash[66969]: audit 2024-03-07T03:48:29.011790+0000 mon.a (mon.0) 1923 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:48:30.613 INFO:tasks.workunit.client.0.smithi145.stdout: i=17 exists [1702466~2087664,5879476~8041759,14637000~8390242,23528317~10427355,35291787~17022528,54498805~15219468,70507758~20524579,95698331~342243,98507705~21910168,122822535~20578415,149370843~9905900,160441040~7060898,168471206~31307322,200362721~18823706,219195368~7248217,226533871~7162347,234880366~14477867,253247316~7099000,261042027~13404274,277821495~25996465,311337650~3081171,314696020~6490561,329129674~6464120,338357992~1253983,341664823~1419274,343892801~34220703,381947474~719903,382802835~16535157,402764712~1521812,404332454~5425555] wrote [8044312~5602073,23528317~5591503,80359445~9330150,178553017~773623,205691423~5897402,219195368~7043729,245609463~3748770,404332454~3843214] 2024-03-07T03:48:30.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:30 smithi145 bash[79878]: cluster 2024-03-07T03:48:29.455144+0000 mon.a (mon.0) 1924 : cluster 0 osdmap e1072: 8 total, 8 up, 8 in 2024-03-07T03:48:30.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:30 smithi145 bash[79878]: cluster 2024-03-07T03:48:29.649631+0000 mgr.y (mgr.34107) 1099 : cluster 0 pgmap v1796: 105 pgs: 105 active+clean; 622 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 214 KiB/s rd, 43 MiB/s wr, 337 op/s 2024-03-07T03:48:30.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:30 smithi145 bash[82329]: cluster 2024-03-07T03:48:29.455144+0000 mon.a (mon.0) 1924 : cluster 0 osdmap e1072: 8 total, 8 up, 8 in 2024-03-07T03:48:30.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:30 smithi145 bash[82329]: cluster 2024-03-07T03:48:29.649631+0000 mgr.y (mgr.34107) 1099 : cluster 0 pgmap v1796: 105 pgs: 105 active+clean; 622 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 214 KiB/s rd, 43 MiB/s wr, 337 op/s 2024-03-07T03:48:30.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:30 smithi189 bash[66969]: cluster 2024-03-07T03:48:29.455144+0000 mon.a (mon.0) 1924 : cluster 0 osdmap e1072: 8 total, 8 up, 8 in 2024-03-07T03:48:30.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:30 smithi189 bash[66969]: cluster 2024-03-07T03:48:29.649631+0000 mgr.y (mgr.34107) 1099 : cluster 0 pgmap v1796: 105 pgs: 105 active+clean; 622 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 214 KiB/s rd, 43 MiB/s wr, 337 op/s 2024-03-07T03:48:31.608 INFO:tasks.workunit.client.0.smithi145.stdout: i=18 exists [1702466~2087664,5879476~8041759,14637000~19318672,35291787~34426486,70507758~20524579,95698331~342243,98507705~21910168,122822535~20578415,149370843~9905900,160441040~7060898,168471206~31307322,200362721~18823706,219195368~7248217,226533871~7162347,234880366~14477867,253247316~21198985,277821495~25996465,311337650~3081171,314696020~6490561,323346509~12247285,338357992~1253983,341664823~1419274,343892801~34220703,381947474~719903,382802835~16535157,402764712~1521812,404332454~5425555] wrote [18472134~7382987,51102699~7078680,177890811~5339529,257545201~67747,259491864~5139870,323346509~7551328,345650652~713154,356705080~2001286,361193735~5902937] 2024-03-07T03:48:31.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:31 smithi145 bash[79878]: cluster 2024-03-07T03:48:30.474829+0000 mon.a (mon.0) 1925 : cluster 0 osdmap e1073: 8 total, 8 up, 8 in 2024-03-07T03:48:31.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:31 smithi145 bash[82329]: cluster 2024-03-07T03:48:30.474829+0000 mon.a (mon.0) 1925 : cluster 0 osdmap e1073: 8 total, 8 up, 8 in 2024-03-07T03:48:31.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:31 smithi189 bash[66969]: cluster 2024-03-07T03:48:30.474829+0000 mon.a (mon.0) 1925 : cluster 0 osdmap e1073: 8 total, 8 up, 8 in 2024-03-07T03:48:32.568 INFO:tasks.workunit.client.0.smithi145.stdout: i=19 exists [1702466~2087664,5879476~8041759,14637000~19318672,35291787~34426486,70507758~20524579,95698331~342243,98507705~21910168,122822535~20578415,149370843~9905900,160441040~7060898,168471206~31307322,200362721~18823706,219195368~7248217,226533871~7162347,234880366~14477867,253247316~21198985,277821495~25996465,311337650~3081171,314696020~6490561,322750577~9859,323346509~12247285,338357992~1253983,341664823~1419274,343892801~34220703,381947474~719903,382802835~16535157,402764712~1521812,404332454~5425555] wrote [5967285~3578983,15971564~2472477,41037600~2454455,85790045~3936357,161761724~3040750,186548911~6176496,322750577~9859,347518675~2153503,362293281~937747] 2024-03-07T03:48:32.588 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:48:32 smithi189 bash[59814]: ts=2024-03-07T03:48:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:48:32.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:32 smithi189 bash[66969]: cluster 2024-03-07T03:48:31.486544+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e1074: 8 total, 8 up, 8 in 2024-03-07T03:48:32.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:32 smithi189 bash[66969]: cluster 2024-03-07T03:48:31.650151+0000 mgr.y (mgr.34107) 1100 : cluster 0 pgmap v1799: 105 pgs: 105 active+clean; 717 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 115 KiB/s rd, 43 MiB/s wr, 206 op/s 2024-03-07T03:48:32.759 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:32 smithi145 bash[82329]: cluster 2024-03-07T03:48:31.486544+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e1074: 8 total, 8 up, 8 in 2024-03-07T03:48:32.760 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:32 smithi145 bash[82329]: cluster 2024-03-07T03:48:31.650151+0000 mgr.y (mgr.34107) 1100 : cluster 0 pgmap v1799: 105 pgs: 105 active+clean; 717 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 115 KiB/s rd, 43 MiB/s wr, 206 op/s 2024-03-07T03:48:32.760 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:32 smithi145 bash[79878]: cluster 2024-03-07T03:48:31.486544+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e1074: 8 total, 8 up, 8 in 2024-03-07T03:48:32.760 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:32 smithi145 bash[79878]: cluster 2024-03-07T03:48:31.650151+0000 mgr.y (mgr.34107) 1100 : cluster 0 pgmap v1799: 105 pgs: 105 active+clean; 717 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 115 KiB/s rd, 43 MiB/s wr, 206 op/s 2024-03-07T03:48:33.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:48:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:48:32] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.43.0" 2024-03-07T03:48:33.531 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 1 diff [50331648~8388608,62914560~8388608,79691776~4194304,109051904~12582912,150994944~8388608,201326592~4194304,310378496~4194304,327155712~4194304,390070272~8388608] 2024-03-07T03:48:33.531 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [50331648~8388608,62914560~8388608,79691776~4194304,109051904~12582912,150994944~8388608,201326592~4194304,310378496~4194304,327155712~4194304,390070272~8388608] 2024-03-07T03:48:33.555 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [50331648~20971520,79691776~4194304,109051904~12582912,150994944~8388608,201326592~4194304,310378496~4194304,327155712~4194304,390070272~8388608] 2024-03-07T03:48:33.555 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 2 diff [12582912~4194304,50331648~8388608,62914560~12582912,79691776~4194304,96468992~4194304,109051904~16777216,150994944~8388608,192937984~12582912,226492416~8388608,297795584~4194304,310378496~4194304,327155712~4194304,335544320~4194304,381681664~4194304,390070272~8388608] 2024-03-07T03:48:33.556 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [12582912~4194304,50331648~8388608,62914560~12582912,79691776~4194304,96468992~4194304,109051904~16777216,150994944~8388608,192937984~12582912,226492416~8388608,297795584~4194304,310378496~4194304,327155712~4194304,335544320~4194304,381681664~4194304,390070272~8388608] 2024-03-07T03:48:33.560 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [12582912~8388608,50331648~25165824,79691776~4194304,96468992~8388608,109051904~16777216,150994944~8388608,192937984~12582912,226492416~8388608,297795584~4194304,310378496~4194304,327155712~4194304,335544320~4194304,381681664~4194304,390070272~8388608] 2024-03-07T03:48:33.560 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 3 diff [12582912~4194304,50331648~8388608,62914560~12582912,79691776~4194304,96468992~4194304,109051904~16777216,138412032~8388608,150994944~8388608,192937984~25165824,226492416~8388608,297795584~4194304,310378496~4194304,327155712~4194304,335544320~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:33.561 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [12582912~4194304,50331648~8388608,62914560~12582912,79691776~4194304,96468992~4194304,109051904~16777216,138412032~8388608,150994944~8388608,192937984~25165824,226492416~8388608,297795584~4194304,310378496~4194304,327155712~4194304,335544320~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:33.567 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [12582912~8388608,50331648~25165824,79691776~4194304,96468992~8388608,109051904~20971520,138412032~8388608,150994944~8388608,192937984~25165824,226492416~8388608,297795584~4194304,310378496~4194304,327155712~4194304,335544320~12582912,381681664~4194304,390070272~8388608] 2024-03-07T03:48:33.567 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 4 diff [12582912~4194304,41943040~16777216,62914560~25165824,96468992~4194304,109051904~16777216,138412032~8388608,150994944~8388608,192937984~29360128,226492416~8388608,251658240~8388608,289406976~8388608,310378496~4194304,327155712~4194304,335544320~8388608,360710144~8388608,381681664~4194304,390070272~8388608,402653184~4194304] 2024-03-07T03:48:33.567 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [12582912~4194304,41943040~16777216,62914560~25165824,96468992~4194304,109051904~16777216,138412032~8388608,150994944~8388608,192937984~29360128,226492416~8388608,251658240~8388608,289406976~8388608,310378496~4194304,327155712~4194304,335544320~8388608,360710144~8388608,381681664~4194304,390070272~8388608,402653184~4194304] 2024-03-07T03:48:33.574 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [12582912~8388608,41943040~46137344,96468992~8388608,109051904~20971520,138412032~8388608,150994944~8388608,192937984~41943040,251658240~12582912,289406976~12582912,310378496~4194304,327155712~4194304,335544320~12582912,360710144~8388608,381681664~4194304,390070272~8388608,402653184~4194304] 2024-03-07T03:48:33.574 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 5 diff [4194304~12582912,37748736~20971520,62914560~25165824,96468992~4194304,109051904~16777216,138412032~8388608,150994944~8388608,180355072~4194304,192937984~20971520,218103808~4194304,226492416~8388608,251658240~8388608,281018368~16777216,310378496~4194304,327155712~4194304,335544320~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.574 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~12582912,37748736~20971520,62914560~25165824,96468992~4194304,109051904~16777216,138412032~8388608,150994944~8388608,180355072~4194304,192937984~20971520,218103808~4194304,226492416~8388608,251658240~8388608,281018368~16777216,310378496~4194304,327155712~4194304,335544320~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.582 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~16777216,37748736~50331648,96468992~8388608,109051904~20971520,138412032~8388608,150994944~8388608,180355072~8388608,192937984~41943040,251658240~12582912,281018368~20971520,310378496~4194304,327155712~4194304,335544320~12582912,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.582 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 6 diff [4194304~12582912,29360128~4194304,37748736~20971520,62914560~25165824,96468992~4194304,104857600~12582912,121634816~4194304,138412032~20971520,180355072~4194304,192937984~20971520,218103808~4194304,226492416~20971520,251658240~8388608,268435456~4194304,281018368~16777216,310378496~8388608,327155712~4194304,335544320~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.582 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~12582912,29360128~4194304,37748736~20971520,62914560~25165824,96468992~4194304,104857600~12582912,121634816~4194304,138412032~20971520,180355072~4194304,192937984~20971520,218103808~4194304,226492416~20971520,251658240~8388608,268435456~4194304,281018368~16777216,310378496~8388608,327155712~4194304,335544320~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.591 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~16777216,29360128~58720256,96468992~33554432,138412032~20971520,180355072~8388608,192937984~54525952,251658240~12582912,268435456~8388608,281018368~20971520,310378496~12582912,327155712~4194304,335544320~12582912,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.591 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 7 diff [4194304~12582912,29360128~50331648,83886080~4194304,96468992~4194304,104857600~12582912,121634816~4194304,138412032~20971520,180355072~4194304,192937984~50331648,251658240~8388608,268435456~4194304,281018368~16777216,310378496~8388608,327155712~4194304,335544320~8388608,348127232~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.591 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~12582912,29360128~50331648,83886080~4194304,96468992~4194304,104857600~12582912,121634816~4194304,138412032~20971520,180355072~4194304,192937984~50331648,251658240~8388608,268435456~4194304,281018368~16777216,310378496~8388608,327155712~4194304,335544320~8388608,348127232~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.601 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~16777216,29360128~58720256,96468992~33554432,138412032~20971520,180355072~8388608,192937984~54525952,251658240~12582912,268435456~8388608,281018368~20971520,310378496~12582912,327155712~4194304,335544320~37748736,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.601 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 8 diff [0~16777216,20971520~4194304,29360128~50331648,83886080~4194304,96468992~4194304,104857600~12582912,121634816~4194304,130023424~12582912,146800640~12582912,180355072~4194304,192937984~50331648,251658240~20971520,281018368~16777216,310378496~12582912,327155712~4194304,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.601 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~16777216,20971520~4194304,29360128~50331648,83886080~4194304,96468992~4194304,104857600~12582912,121634816~4194304,130023424~12582912,146800640~12582912,180355072~4194304,192937984~50331648,251658240~20971520,281018368~16777216,310378496~12582912,327155712~4194304,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.612 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~25165824,29360128~58720256,96468992~62914560,180355072~8388608,192937984~54525952,251658240~25165824,281018368~20971520,310378496~12582912,327155712~4194304,335544320~37748736,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.612 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 9 diff [0~16777216,20971520~58720256,83886080~4194304,96468992~20971520,121634816~4194304,130023424~12582912,146800640~12582912,180355072~4194304,192937984~50331648,251658240~20971520,276824064~25165824,310378496~12582912,327155712~4194304,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.612 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~16777216,20971520~58720256,83886080~4194304,96468992~20971520,121634816~4194304,130023424~12582912,146800640~12582912,180355072~4194304,192937984~50331648,251658240~20971520,276824064~25165824,310378496~12582912,327155712~4194304,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.624 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~88080384,96468992~62914560,180355072~8388608,192937984~54525952,251658240~54525952,310378496~12582912,327155712~4194304,335544320~37748736,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.624 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 10 diff [0~16777216,20971520~62914560,96468992~29360128,130023424~12582912,146800640~12582912,180355072~8388608,192937984~50331648,251658240~20971520,276824064~25165824,310378496~12582912,327155712~4194304,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.625 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~16777216,20971520~62914560,96468992~29360128,130023424~12582912,146800640~12582912,180355072~8388608,192937984~50331648,251658240~20971520,276824064~25165824,310378496~12582912,327155712~4194304,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.638 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~88080384,96468992~62914560,180355072~8388608,192937984~54525952,251658240~54525952,310378496~12582912,327155712~4194304,335544320~37748736,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.638 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 11 diff [0~88080384,96468992~29360128,130023424~12582912,146800640~12582912,180355072~8388608,192937984~50331648,251658240~20971520,276824064~25165824,310378496~12582912,327155712~4194304,335544320~20971520,360710144~12582912,381681664~25165824] 2024-03-07T03:48:33.638 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~29360128,130023424~12582912,146800640~12582912,180355072~8388608,192937984~50331648,251658240~20971520,276824064~25165824,310378496~12582912,327155712~4194304,335544320~20971520,360710144~12582912,381681664~25165824] 2024-03-07T03:48:33.653 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~62914560,180355072~8388608,192937984~54525952,251658240~54525952,310378496~12582912,327155712~4194304,335544320~37748736,381681664~25165824] 2024-03-07T03:48:33.653 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 12 diff [0~88080384,96468992~29360128,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~50331648,251658240~50331648,310378496~12582912,327155712~4194304,335544320~20971520,360710144~12582912,381681664~25165824] 2024-03-07T03:48:33.654 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~29360128,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~50331648,251658240~50331648,310378496~12582912,327155712~4194304,335544320~20971520,360710144~12582912,381681664~25165824] 2024-03-07T03:48:33.669 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~62914560,167772160~8388608,180355072~8388608,192937984~54525952,251658240~54525952,310378496~12582912,327155712~4194304,335544320~37748736,381681664~25165824] 2024-03-07T03:48:33.669 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 13 diff [0~88080384,96468992~29360128,130023424~12582912,146800640~41943040,192937984~50331648,251658240~54525952,310378496~12582912,327155712~4194304,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.669 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~29360128,130023424~12582912,146800640~41943040,192937984~50331648,251658240~54525952,310378496~12582912,327155712~4194304,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.685 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~150994944,251658240~54525952,310378496~12582912,327155712~4194304,335544320~37748736,381681664~25165824] 2024-03-07T03:48:33.685 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 14 diff [0~88080384,96468992~46137344,146800640~41943040,192937984~50331648,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.685 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~46137344,146800640~41943040,192937984~50331648,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.703 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~150994944,251658240~54525952,310378496~12582912,327155712~79691776] 2024-03-07T03:48:33.703 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 15 diff [0~88080384,96468992~46137344,146800640~96468992,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.703 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~46137344,146800640~96468992,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.721 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~150994944,251658240~54525952,310378496~12582912,327155712~79691776] 2024-03-07T03:48:33.722 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 16 diff [0~88080384,92274688~50331648,146800640~96468992,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~8388608] 2024-03-07T03:48:33.722 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~96468992,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~8388608] 2024-03-07T03:48:33.742 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~247463936,251658240~54525952,310378496~12582912,327155712~83886080] 2024-03-07T03:48:33.742 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 17 diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~8388608] 2024-03-07T03:48:33.743 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~8388608] 2024-03-07T03:48:33.764 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~12582912,327155712~83886080] 2024-03-07T03:48:33.764 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 18 diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:33.764 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:33.784 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:33.785 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to 19 diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:33.785 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:33.808 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:33.808 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 2 diff [12582912~4194304,71303168~4194304,96468992~4194304,121634816~4194304,192937984~8388608,226492416~8388608,297795584~4194304,335544320~4194304,381681664~4194304] 2024-03-07T03:48:33.808 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [12582912~4194304,71303168~4194304,96468992~4194304,121634816~4194304,192937984~8388608,226492416~8388608,297795584~4194304,335544320~4194304,381681664~4194304] 2024-03-07T03:48:33.811 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [12582912~8388608,71303168~4194304,96468992~8388608,121634816~4194304,192937984~8388608,226492416~8388608,297795584~4194304,335544320~4194304,381681664~4194304] 2024-03-07T03:48:33.811 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 3 diff [12582912~4194304,71303168~4194304,96468992~4194304,121634816~4194304,138412032~8388608,192937984~25165824,226492416~8388608,297795584~4194304,310378496~4194304,335544320~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:33.811 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [12582912~4194304,71303168~4194304,96468992~4194304,121634816~4194304,138412032~8388608,192937984~25165824,226492416~8388608,297795584~4194304,310378496~4194304,335544320~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:33.816 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [12582912~8388608,71303168~4194304,96468992~8388608,121634816~8388608,138412032~8388608,192937984~25165824,226492416~8388608,297795584~4194304,310378496~4194304,335544320~12582912,381681664~4194304,390070272~8388608] 2024-03-07T03:48:33.816 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 4 diff [12582912~4194304,41943040~8388608,71303168~16777216,96468992~4194304,121634816~4194304,138412032~8388608,155189248~4194304,192937984~29360128,226492416~8388608,251658240~8388608,289406976~8388608,310378496~4194304,335544320~8388608,360710144~8388608,381681664~4194304,390070272~8388608,402653184~4194304] 2024-03-07T03:48:33.816 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [12582912~4194304,41943040~8388608,71303168~16777216,96468992~4194304,121634816~4194304,138412032~8388608,155189248~4194304,192937984~29360128,226492416~8388608,251658240~8388608,289406976~8388608,310378496~4194304,335544320~8388608,360710144~8388608,381681664~4194304,390070272~8388608,402653184~4194304] 2024-03-07T03:48:33.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:33 smithi145 bash[79878]: cluster 2024-03-07T03:48:32.498324+0000 mon.a (mon.0) 1927 : cluster 0 osdmap e1075: 8 total, 8 up, 8 in 2024-03-07T03:48:33.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:33 smithi145 bash[79878]: audit 2024-03-07T03:48:32.682609+0000 mon.a (mon.0) 1928 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:33.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:33 smithi145 bash[82329]: cluster 2024-03-07T03:48:32.498324+0000 mon.a (mon.0) 1927 : cluster 0 osdmap e1075: 8 total, 8 up, 8 in 2024-03-07T03:48:33.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:33 smithi145 bash[82329]: audit 2024-03-07T03:48:32.682609+0000 mon.a (mon.0) 1928 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:33.822 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [12582912~8388608,41943040~8388608,71303168~16777216,96468992~8388608,121634816~8388608,138412032~8388608,155189248~4194304,192937984~41943040,251658240~12582912,289406976~12582912,310378496~4194304,335544320~12582912,360710144~8388608,381681664~4194304,390070272~8388608,402653184~4194304] 2024-03-07T03:48:33.822 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 5 diff [4194304~12582912,37748736~12582912,71303168~16777216,96468992~4194304,121634816~4194304,138412032~8388608,155189248~4194304,180355072~4194304,192937984~20971520,218103808~4194304,226492416~8388608,251658240~8388608,281018368~16777216,310378496~4194304,335544320~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.822 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~12582912,37748736~12582912,71303168~16777216,96468992~4194304,121634816~4194304,138412032~8388608,155189248~4194304,180355072~4194304,192937984~20971520,218103808~4194304,226492416~8388608,251658240~8388608,281018368~16777216,310378496~4194304,335544320~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.828 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~16777216,37748736~12582912,71303168~16777216,96468992~8388608,121634816~8388608,138412032~8388608,155189248~4194304,180355072~8388608,192937984~41943040,251658240~12582912,281018368~20971520,310378496~4194304,335544320~12582912,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.829 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 6 diff [4194304~12582912,29360128~4194304,37748736~12582912,71303168~16777216,96468992~4194304,104857600~4194304,121634816~4194304,138412032~12582912,155189248~4194304,180355072~4194304,192937984~20971520,218103808~4194304,226492416~20971520,251658240~8388608,268435456~4194304,281018368~16777216,310378496~8388608,335544320~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.829 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~12582912,29360128~4194304,37748736~12582912,71303168~16777216,96468992~4194304,104857600~4194304,121634816~4194304,138412032~12582912,155189248~4194304,180355072~4194304,192937984~20971520,218103808~4194304,226492416~20971520,251658240~8388608,268435456~4194304,281018368~16777216,310378496~8388608,335544320~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.836 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~16777216,29360128~20971520,71303168~16777216,96468992~16777216,121634816~8388608,138412032~12582912,155189248~4194304,180355072~8388608,192937984~54525952,251658240~12582912,268435456~8388608,281018368~20971520,310378496~12582912,335544320~12582912,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.836 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 7 diff [4194304~12582912,29360128~20971520,58720256~20971520,83886080~4194304,96468992~4194304,104857600~4194304,121634816~4194304,138412032~12582912,155189248~4194304,180355072~4194304,192937984~50331648,251658240~8388608,268435456~4194304,281018368~16777216,310378496~8388608,335544320~8388608,348127232~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.836 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~12582912,29360128~20971520,58720256~20971520,83886080~4194304,96468992~4194304,104857600~4194304,121634816~4194304,138412032~12582912,155189248~4194304,180355072~4194304,192937984~50331648,251658240~8388608,268435456~4194304,281018368~16777216,310378496~8388608,335544320~8388608,348127232~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:33 smithi189 bash[66969]: cluster 2024-03-07T03:48:32.498324+0000 mon.a (mon.0) 1927 : cluster 0 osdmap e1075: 8 total, 8 up, 8 in 2024-03-07T03:48:33.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:33 smithi189 bash[66969]: audit 2024-03-07T03:48:32.682609+0000 mon.a (mon.0) 1928 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:33.844 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~16777216,29360128~20971520,58720256~29360128,96468992~16777216,121634816~8388608,138412032~12582912,155189248~4194304,180355072~8388608,192937984~54525952,251658240~12582912,268435456~8388608,281018368~20971520,310378496~12582912,335544320~37748736,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.845 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 8 diff [0~16777216,20971520~4194304,29360128~20971520,58720256~20971520,83886080~4194304,96468992~4194304,104857600~4194304,121634816~4194304,130023424~12582912,146800640~4194304,155189248~4194304,180355072~4194304,192937984~50331648,251658240~20971520,281018368~16777216,310378496~12582912,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.845 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~16777216,20971520~4194304,29360128~20971520,58720256~20971520,83886080~4194304,96468992~4194304,104857600~4194304,121634816~4194304,130023424~12582912,146800640~4194304,155189248~4194304,180355072~4194304,192937984~50331648,251658240~20971520,281018368~16777216,310378496~12582912,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.855 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~25165824,29360128~25165824,58720256~29360128,96468992~16777216,121634816~29360128,155189248~4194304,180355072~8388608,192937984~54525952,251658240~25165824,281018368~20971520,310378496~12582912,335544320~37748736,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.856 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 9 diff [0~16777216,20971520~29360128,58720256~20971520,83886080~4194304,96468992~12582912,121634816~4194304,130023424~12582912,146800640~4194304,155189248~4194304,180355072~4194304,192937984~50331648,251658240~20971520,276824064~25165824,310378496~12582912,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.856 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~16777216,20971520~29360128,58720256~20971520,83886080~4194304,96468992~12582912,121634816~4194304,130023424~12582912,146800640~4194304,155189248~4194304,180355072~4194304,192937984~50331648,251658240~20971520,276824064~25165824,310378496~12582912,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.867 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~54525952,58720256~29360128,96468992~16777216,121634816~29360128,155189248~4194304,180355072~8388608,192937984~54525952,251658240~54525952,310378496~12582912,335544320~37748736,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.867 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 10 diff [0~16777216,20971520~29360128,58720256~25165824,96468992~16777216,121634816~4194304,130023424~12582912,146800640~12582912,180355072~8388608,192937984~50331648,251658240~20971520,276824064~25165824,310378496~12582912,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.867 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~16777216,20971520~29360128,58720256~25165824,96468992~16777216,121634816~4194304,130023424~12582912,146800640~12582912,180355072~8388608,192937984~50331648,251658240~20971520,276824064~25165824,310378496~12582912,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.879 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~54525952,58720256~29360128,96468992~16777216,121634816~37748736,180355072~8388608,192937984~54525952,251658240~54525952,310378496~12582912,335544320~37748736,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.879 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 11 diff [0~50331648,54525952~33554432,96468992~16777216,121634816~4194304,130023424~12582912,146800640~12582912,180355072~8388608,192937984~50331648,251658240~20971520,276824064~25165824,310378496~12582912,335544320~20971520,360710144~12582912,381681664~25165824] 2024-03-07T03:48:33.879 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~50331648,54525952~33554432,96468992~16777216,121634816~4194304,130023424~12582912,146800640~12582912,180355072~8388608,192937984~50331648,251658240~20971520,276824064~25165824,310378496~12582912,335544320~20971520,360710144~12582912,381681664~25165824] 2024-03-07T03:48:33.891 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,121634816~37748736,180355072~8388608,192937984~54525952,251658240~54525952,310378496~12582912,335544320~37748736,381681664~25165824] 2024-03-07T03:48:33.892 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 12 diff [0~88080384,96468992~16777216,121634816~4194304,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~50331648,251658240~50331648,310378496~12582912,335544320~20971520,360710144~12582912,381681664~25165824] 2024-03-07T03:48:33.892 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~16777216,121634816~4194304,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~50331648,251658240~50331648,310378496~12582912,335544320~20971520,360710144~12582912,381681664~25165824] 2024-03-07T03:48:33.906 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,121634816~37748736,167772160~8388608,180355072~8388608,192937984~54525952,251658240~54525952,310378496~12582912,335544320~37748736,381681664~25165824] 2024-03-07T03:48:33.907 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 13 diff [0~88080384,96468992~16777216,121634816~4194304,130023424~12582912,146800640~41943040,192937984~50331648,251658240~54525952,310378496~12582912,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.907 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~16777216,121634816~4194304,130023424~12582912,146800640~41943040,192937984~50331648,251658240~54525952,310378496~12582912,335544320~20971520,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.921 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,121634816~125829120,251658240~54525952,310378496~12582912,335544320~37748736,381681664~25165824] 2024-03-07T03:48:33.921 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 14 diff [0~88080384,96468992~16777216,121634816~20971520,146800640~41943040,192937984~50331648,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.921 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~16777216,121634816~20971520,146800640~41943040,192937984~50331648,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.938 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,121634816~125829120,251658240~54525952,310378496~12582912,327155712~79691776] 2024-03-07T03:48:33.939 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 15 diff [0~88080384,96468992~12582912,117440512~25165824,146800640~96468992,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.939 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~12582912,117440512~25165824,146800640~96468992,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~4194304] 2024-03-07T03:48:33.956 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,117440512~130023424,251658240~54525952,310378496~12582912,327155712~79691776] 2024-03-07T03:48:33.957 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 16 diff [0~88080384,92274688~20971520,117440512~25165824,146800640~96468992,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~8388608] 2024-03-07T03:48:33.957 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~96468992,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~8388608] 2024-03-07T03:48:33.974 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~247463936,251658240~54525952,310378496~12582912,327155712~83886080] 2024-03-07T03:48:33.974 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 17 diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~8388608] 2024-03-07T03:48:33.974 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~12582912,327155712~50331648,381681664~16777216,402653184~8388608] 2024-03-07T03:48:33.992 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~12582912,327155712~83886080] 2024-03-07T03:48:33.992 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 18 diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:33.992 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.013 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:34.014 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to 19 diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.014 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.035 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:34.036 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 3 diff [121634816~4194304,138412032~8388608,197132288~20971520,310378496~4194304,339738624~4194304,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.036 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [121634816~4194304,138412032~8388608,197132288~20971520,310378496~4194304,339738624~4194304,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.039 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [121634816~8388608,138412032~8388608,197132288~20971520,310378496~4194304,339738624~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.039 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 4 diff [12582912~4194304,41943040~8388608,75497472~12582912,121634816~4194304,138412032~8388608,155189248~4194304,197132288~25165824,251658240~8388608,289406976~8388608,310378496~4194304,339738624~4194304,360710144~8388608,381681664~4194304,390070272~8388608,402653184~4194304] 2024-03-07T03:48:34.039 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [12582912~4194304,41943040~8388608,75497472~12582912,121634816~4194304,138412032~8388608,155189248~4194304,197132288~25165824,251658240~8388608,289406976~8388608,310378496~4194304,339738624~4194304,360710144~8388608,381681664~4194304,390070272~8388608,402653184~4194304] 2024-03-07T03:48:34.044 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [12582912~8388608,41943040~8388608,75497472~12582912,121634816~8388608,138412032~8388608,155189248~4194304,197132288~29360128,251658240~12582912,289406976~12582912,310378496~4194304,339738624~8388608,360710144~8388608,381681664~4194304,390070272~8388608,402653184~4194304] 2024-03-07T03:48:34.044 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 5 diff [4194304~12582912,37748736~12582912,75497472~12582912,121634816~4194304,138412032~8388608,155189248~4194304,180355072~4194304,197132288~16777216,218103808~4194304,251658240~8388608,281018368~16777216,310378496~4194304,339738624~4194304,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.044 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~12582912,37748736~12582912,75497472~12582912,121634816~4194304,138412032~8388608,155189248~4194304,180355072~4194304,197132288~16777216,218103808~4194304,251658240~8388608,281018368~16777216,310378496~4194304,339738624~4194304,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.049 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~16777216,37748736~12582912,75497472~12582912,121634816~8388608,138412032~8388608,155189248~4194304,180355072~8388608,197132288~29360128,251658240~12582912,281018368~20971520,310378496~4194304,339738624~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.049 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 6 diff [4194304~12582912,29360128~4194304,37748736~12582912,75497472~12582912,104857600~4194304,121634816~4194304,138412032~12582912,155189248~4194304,180355072~4194304,197132288~16777216,218103808~4194304,234881024~12582912,251658240~8388608,268435456~4194304,281018368~16777216,310378496~8388608,339738624~4194304,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.050 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~12582912,29360128~4194304,37748736~12582912,75497472~12582912,104857600~4194304,121634816~4194304,138412032~12582912,155189248~4194304,180355072~4194304,197132288~16777216,218103808~4194304,234881024~12582912,251658240~8388608,268435456~4194304,281018368~16777216,310378496~8388608,339738624~4194304,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.056 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~16777216,29360128~20971520,75497472~12582912,104857600~8388608,121634816~8388608,138412032~12582912,155189248~4194304,180355072~8388608,197132288~29360128,234881024~12582912,251658240~12582912,268435456~8388608,281018368~20971520,310378496~12582912,339738624~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.056 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 7 diff [4194304~12582912,29360128~20971520,58720256~20971520,83886080~4194304,104857600~4194304,121634816~4194304,138412032~12582912,155189248~4194304,180355072~4194304,197132288~29360128,230686720~12582912,251658240~8388608,268435456~4194304,281018368~16777216,310378496~8388608,339738624~4194304,348127232~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.056 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~12582912,29360128~20971520,58720256~20971520,83886080~4194304,104857600~4194304,121634816~4194304,138412032~12582912,155189248~4194304,180355072~4194304,197132288~29360128,230686720~12582912,251658240~8388608,268435456~4194304,281018368~16777216,310378496~8388608,339738624~4194304,348127232~8388608,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.063 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~16777216,29360128~20971520,58720256~29360128,104857600~8388608,121634816~8388608,138412032~12582912,155189248~4194304,180355072~8388608,197132288~29360128,230686720~16777216,251658240~12582912,268435456~8388608,281018368~20971520,310378496~12582912,339738624~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.063 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 8 diff [0~16777216,20971520~4194304,29360128~20971520,58720256~20971520,83886080~4194304,104857600~4194304,121634816~4194304,130023424~12582912,146800640~4194304,155189248~4194304,180355072~4194304,197132288~29360128,230686720~12582912,251658240~20971520,281018368~16777216,310378496~12582912,339738624~16777216,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.063 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~16777216,20971520~4194304,29360128~20971520,58720256~20971520,83886080~4194304,104857600~4194304,121634816~4194304,130023424~12582912,146800640~4194304,155189248~4194304,180355072~4194304,197132288~29360128,230686720~12582912,251658240~20971520,281018368~16777216,310378496~12582912,339738624~16777216,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.073 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~25165824,29360128~25165824,58720256~29360128,104857600~8388608,121634816~29360128,155189248~4194304,180355072~8388608,197132288~29360128,230686720~16777216,251658240~25165824,281018368~20971520,310378496~12582912,339738624~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.073 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 9 diff [0~16777216,20971520~29360128,58720256~20971520,83886080~4194304,96468992~12582912,121634816~4194304,130023424~12582912,146800640~4194304,155189248~4194304,180355072~4194304,197132288~29360128,230686720~12582912,251658240~20971520,276824064~25165824,310378496~12582912,339738624~16777216,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.073 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~16777216,20971520~29360128,58720256~20971520,83886080~4194304,96468992~12582912,121634816~4194304,130023424~12582912,146800640~4194304,155189248~4194304,180355072~4194304,197132288~29360128,230686720~12582912,251658240~20971520,276824064~25165824,310378496~12582912,339738624~16777216,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.083 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~54525952,58720256~29360128,96468992~16777216,121634816~29360128,155189248~4194304,180355072~8388608,197132288~29360128,230686720~16777216,251658240~54525952,310378496~12582912,339738624~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.083 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 10 diff [0~16777216,20971520~29360128,58720256~25165824,96468992~16777216,121634816~4194304,130023424~12582912,146800640~12582912,180355072~8388608,192937984~33554432,230686720~12582912,251658240~20971520,276824064~25165824,310378496~12582912,339738624~16777216,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.083 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~16777216,20971520~29360128,58720256~25165824,96468992~16777216,121634816~4194304,130023424~12582912,146800640~12582912,180355072~8388608,192937984~33554432,230686720~12582912,251658240~20971520,276824064~25165824,310378496~12582912,339738624~16777216,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.094 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~54525952,58720256~29360128,96468992~16777216,121634816~37748736,180355072~8388608,192937984~33554432,230686720~16777216,251658240~54525952,310378496~12582912,339738624~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.094 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 11 diff [0~50331648,54525952~33554432,96468992~16777216,121634816~4194304,130023424~12582912,146800640~12582912,180355072~8388608,192937984~33554432,230686720~12582912,251658240~20971520,276824064~25165824,310378496~12582912,339738624~16777216,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.094 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~50331648,54525952~33554432,96468992~16777216,121634816~4194304,130023424~12582912,146800640~12582912,180355072~8388608,192937984~33554432,230686720~12582912,251658240~20971520,276824064~25165824,310378496~12582912,339738624~16777216,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.106 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,121634816~37748736,180355072~8388608,192937984~33554432,230686720~16777216,251658240~54525952,310378496~12582912,339738624~33554432,381681664~25165824] 2024-03-07T03:48:34.106 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 12 diff [0~88080384,96468992~16777216,121634816~4194304,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~33554432,230686720~12582912,251658240~50331648,310378496~12582912,339738624~16777216,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.106 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~16777216,121634816~4194304,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~33554432,230686720~12582912,251658240~50331648,310378496~12582912,339738624~16777216,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.120 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,121634816~37748736,167772160~8388608,180355072~8388608,192937984~33554432,230686720~16777216,251658240~54525952,310378496~12582912,339738624~33554432,381681664~25165824] 2024-03-07T03:48:34.120 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 13 diff [0~88080384,96468992~16777216,121634816~4194304,130023424~12582912,146800640~41943040,192937984~33554432,230686720~12582912,251658240~54525952,310378496~12582912,339738624~16777216,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.120 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~16777216,121634816~4194304,130023424~12582912,146800640~41943040,192937984~33554432,230686720~12582912,251658240~54525952,310378496~12582912,339738624~16777216,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.134 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,121634816~104857600,230686720~16777216,251658240~54525952,310378496~12582912,339738624~33554432,381681664~25165824] 2024-03-07T03:48:34.134 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 14 diff [0~88080384,96468992~16777216,121634816~20971520,146800640~41943040,192937984~33554432,230686720~12582912,251658240~54525952,310378496~12582912,327155712~8388608,339738624~37748736,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.134 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~16777216,121634816~20971520,146800640~41943040,192937984~33554432,230686720~12582912,251658240~54525952,310378496~12582912,327155712~8388608,339738624~37748736,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.150 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,121634816~104857600,230686720~16777216,251658240~54525952,310378496~12582912,327155712~79691776] 2024-03-07T03:48:34.150 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 15 diff [0~88080384,96468992~12582912,117440512~25165824,146800640~79691776,230686720~12582912,251658240~54525952,310378496~12582912,327155712~8388608,339738624~37748736,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.150 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~12582912,117440512~25165824,146800640~79691776,230686720~12582912,251658240~54525952,310378496~12582912,327155712~8388608,339738624~37748736,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.166 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,117440512~109051904,230686720~16777216,251658240~54525952,310378496~12582912,327155712~79691776] 2024-03-07T03:48:34.166 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 16 diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~12582912,251658240~54525952,310378496~12582912,327155712~8388608,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.166 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~12582912,251658240~54525952,310378496~12582912,327155712~8388608,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.184 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~16777216,251658240~54525952,310378496~12582912,327155712~83886080] 2024-03-07T03:48:34.184 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 17 diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~12582912,327155712~8388608,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.184 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~12582912,327155712~8388608,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.203 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~12582912,327155712~83886080] 2024-03-07T03:48:34.203 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 18 diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.203 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.222 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:34.223 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to 19 diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.223 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.243 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:34.243 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 4 diff [12582912~4194304,41943040~8388608,75497472~12582912,155189248~4194304,218103808~4194304,251658240~8388608,289406976~8388608,360710144~8388608,402653184~4194304] 2024-03-07T03:48:34.243 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [12582912~4194304,41943040~8388608,75497472~12582912,155189248~4194304,218103808~4194304,251658240~8388608,289406976~8388608,360710144~8388608,402653184~4194304] 2024-03-07T03:48:34.246 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [12582912~8388608,41943040~8388608,75497472~12582912,155189248~4194304,218103808~8388608,251658240~12582912,289406976~12582912,360710144~8388608,402653184~4194304] 2024-03-07T03:48:34.246 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 5 diff [4194304~12582912,37748736~12582912,75497472~12582912,155189248~4194304,180355072~4194304,201326592~8388608,218103808~4194304,251658240~8388608,281018368~16777216,360710144~12582912,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.247 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~12582912,37748736~12582912,75497472~12582912,155189248~4194304,180355072~4194304,201326592~8388608,218103808~4194304,251658240~8388608,281018368~16777216,360710144~12582912,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.250 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~16777216,37748736~12582912,75497472~12582912,155189248~4194304,180355072~8388608,201326592~8388608,218103808~8388608,251658240~12582912,281018368~20971520,360710144~12582912,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.250 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 6 diff [4194304~12582912,29360128~4194304,37748736~12582912,75497472~12582912,104857600~4194304,146800640~4194304,155189248~4194304,180355072~4194304,201326592~12582912,218103808~4194304,234881024~12582912,251658240~8388608,268435456~4194304,281018368~16777216,314572800~4194304,360710144~12582912,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.251 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~12582912,29360128~4194304,37748736~12582912,75497472~12582912,104857600~4194304,146800640~4194304,155189248~4194304,180355072~4194304,201326592~12582912,218103808~4194304,234881024~12582912,251658240~8388608,268435456~4194304,281018368~16777216,314572800~4194304,360710144~12582912,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.256 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~16777216,29360128~20971520,75497472~12582912,104857600~8388608,146800640~4194304,155189248~4194304,180355072~8388608,201326592~12582912,218103808~8388608,234881024~12582912,251658240~12582912,268435456~8388608,281018368~20971520,314572800~8388608,360710144~12582912,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.256 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 7 diff [4194304~12582912,29360128~20971520,58720256~20971520,83886080~4194304,104857600~4194304,146800640~4194304,155189248~4194304,180355072~4194304,201326592~25165824,230686720~12582912,251658240~8388608,268435456~4194304,281018368~16777216,314572800~4194304,348127232~8388608,360710144~12582912,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.256 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~12582912,29360128~20971520,58720256~20971520,83886080~4194304,104857600~4194304,146800640~4194304,155189248~4194304,180355072~4194304,201326592~25165824,230686720~12582912,251658240~8388608,268435456~4194304,281018368~16777216,314572800~4194304,348127232~8388608,360710144~12582912,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.262 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~16777216,29360128~20971520,58720256~29360128,104857600~8388608,146800640~4194304,155189248~4194304,180355072~8388608,201326592~25165824,230686720~16777216,251658240~12582912,268435456~8388608,281018368~20971520,314572800~8388608,348127232~25165824,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.262 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 8 diff [0~16777216,20971520~4194304,29360128~20971520,58720256~20971520,83886080~4194304,104857600~4194304,130023424~8388608,146800640~4194304,155189248~4194304,180355072~4194304,201326592~25165824,230686720~12582912,251658240~20971520,281018368~16777216,314572800~8388608,343932928~12582912,360710144~12582912,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.262 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~16777216,20971520~4194304,29360128~20971520,58720256~20971520,83886080~4194304,104857600~4194304,130023424~8388608,146800640~4194304,155189248~4194304,180355072~4194304,201326592~25165824,230686720~12582912,251658240~20971520,281018368~16777216,314572800~8388608,343932928~12582912,360710144~12582912,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.269 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~25165824,29360128~25165824,58720256~29360128,104857600~8388608,130023424~12582912,146800640~4194304,155189248~4194304,180355072~8388608,201326592~25165824,230686720~16777216,251658240~25165824,281018368~20971520,314572800~8388608,343932928~29360128,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.269 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 9 diff [0~16777216,20971520~29360128,58720256~20971520,83886080~4194304,96468992~12582912,130023424~8388608,146800640~4194304,155189248~4194304,180355072~4194304,201326592~25165824,230686720~12582912,251658240~20971520,276824064~25165824,314572800~8388608,343932928~12582912,360710144~12582912,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.269 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~16777216,20971520~29360128,58720256~20971520,83886080~4194304,96468992~12582912,130023424~8388608,146800640~4194304,155189248~4194304,180355072~4194304,201326592~25165824,230686720~12582912,251658240~20971520,276824064~25165824,314572800~8388608,343932928~12582912,360710144~12582912,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.277 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~54525952,58720256~29360128,96468992~16777216,130023424~12582912,146800640~4194304,155189248~4194304,180355072~8388608,201326592~25165824,230686720~16777216,251658240~54525952,314572800~8388608,343932928~29360128,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.277 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 10 diff [0~16777216,20971520~29360128,58720256~25165824,96468992~16777216,130023424~8388608,146800640~12582912,180355072~8388608,192937984~4194304,201326592~25165824,230686720~12582912,251658240~20971520,276824064~25165824,314572800~8388608,343932928~12582912,360710144~12582912,381681664~12582912,402653184~4194304] 2024-03-07T03:48:34.277 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~16777216,20971520~29360128,58720256~25165824,96468992~16777216,130023424~8388608,146800640~12582912,180355072~8388608,192937984~4194304,201326592~25165824,230686720~12582912,251658240~20971520,276824064~25165824,314572800~8388608,343932928~12582912,360710144~12582912,381681664~12582912,402653184~4194304] 2024-03-07T03:48:34.286 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~54525952,58720256~29360128,96468992~16777216,130023424~12582912,146800640~12582912,180355072~8388608,192937984~4194304,201326592~25165824,230686720~16777216,251658240~54525952,314572800~8388608,343932928~29360128,381681664~12582912,402653184~4194304] 2024-03-07T03:48:34.286 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 11 diff [0~50331648,54525952~33554432,96468992~16777216,130023424~12582912,146800640~12582912,180355072~8388608,192937984~4194304,201326592~25165824,230686720~12582912,251658240~20971520,276824064~25165824,314572800~8388608,343932928~12582912,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.286 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~50331648,54525952~33554432,96468992~16777216,130023424~12582912,146800640~12582912,180355072~8388608,192937984~4194304,201326592~25165824,230686720~12582912,251658240~20971520,276824064~25165824,314572800~8388608,343932928~12582912,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.296 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,130023424~12582912,146800640~12582912,180355072~8388608,192937984~4194304,201326592~25165824,230686720~16777216,251658240~54525952,314572800~8388608,343932928~29360128,381681664~25165824] 2024-03-07T03:48:34.296 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 12 diff [0~88080384,96468992~16777216,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~4194304,201326592~25165824,230686720~12582912,251658240~50331648,314572800~8388608,343932928~12582912,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.296 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~16777216,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~4194304,201326592~25165824,230686720~12582912,251658240~50331648,314572800~8388608,343932928~12582912,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.308 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~4194304,201326592~25165824,230686720~16777216,251658240~54525952,314572800~8388608,343932928~29360128,381681664~25165824] 2024-03-07T03:48:34.308 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 13 diff [0~88080384,96468992~16777216,130023424~12582912,146800640~41943040,192937984~4194304,201326592~25165824,230686720~12582912,251658240~54525952,314572800~8388608,343932928~12582912,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.308 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~16777216,130023424~12582912,146800640~41943040,192937984~4194304,201326592~25165824,230686720~12582912,251658240~54525952,314572800~8388608,343932928~12582912,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.320 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,130023424~12582912,146800640~50331648,201326592~25165824,230686720~16777216,251658240~54525952,314572800~8388608,343932928~29360128,381681664~25165824] 2024-03-07T03:48:34.320 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 14 diff [0~88080384,96468992~16777216,125829120~16777216,146800640~41943040,192937984~4194304,201326592~25165824,230686720~12582912,251658240~54525952,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.320 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~16777216,125829120~16777216,146800640~41943040,192937984~4194304,201326592~25165824,230686720~12582912,251658240~54525952,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.334 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,125829120~16777216,146800640~50331648,201326592~25165824,230686720~16777216,251658240~54525952,314572800~8388608,327155712~12582912,343932928~62914560] 2024-03-07T03:48:34.334 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 15 diff [0~88080384,96468992~12582912,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~12582912,251658240~54525952,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.334 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,96468992~12582912,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~12582912,251658240~54525952,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.348 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~16777216,251658240~54525952,314572800~8388608,327155712~12582912,343932928~62914560] 2024-03-07T03:48:34.348 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 16 diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~12582912,251658240~54525952,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.348 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~12582912,251658240~54525952,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.363 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~8388608,327155712~12582912,343932928~67108864] 2024-03-07T03:48:34.363 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 17 diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~16777216,251658240~54525952,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.363 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~16777216,251658240~54525952,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.379 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~8388608,327155712~12582912,343932928~67108864] 2024-03-07T03:48:34.379 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 18 diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.379 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.396 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:34.397 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to 19 diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.397 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.416 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:34.416 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 5 diff [4194304~8388608,37748736~4194304,180355072~4194304,201326592~8388608,281018368~12582912,369098752~4194304,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.416 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,37748736~4194304,180355072~4194304,201326592~8388608,281018368~12582912,369098752~4194304,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.419 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~12582912,37748736~4194304,180355072~8388608,201326592~8388608,281018368~16777216,369098752~4194304,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.419 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 6 diff [4194304~8388608,29360128~4194304,37748736~4194304,104857600~4194304,146800640~4194304,180355072~4194304,201326592~12582912,234881024~12582912,268435456~4194304,281018368~12582912,314572800~4194304,369098752~4194304,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.420 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,29360128~4194304,37748736~4194304,104857600~4194304,146800640~4194304,180355072~4194304,201326592~12582912,234881024~12582912,268435456~4194304,281018368~12582912,314572800~4194304,369098752~4194304,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.423 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~12582912,29360128~12582912,104857600~8388608,146800640~4194304,180355072~8388608,201326592~12582912,234881024~12582912,268435456~8388608,281018368~16777216,314572800~8388608,369098752~4194304,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.423 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 7 diff [4194304~8388608,29360128~12582912,58720256~16777216,104857600~4194304,146800640~4194304,180355072~4194304,201326592~20971520,230686720~12582912,268435456~4194304,281018368~12582912,314572800~4194304,348127232~8388608,364904448~8388608,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.424 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,29360128~12582912,58720256~16777216,104857600~4194304,146800640~4194304,180355072~4194304,201326592~20971520,230686720~12582912,268435456~4194304,281018368~12582912,314572800~4194304,348127232~8388608,364904448~8388608,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.428 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~12582912,29360128~12582912,58720256~20971520,104857600~8388608,146800640~4194304,180355072~8388608,201326592~25165824,230686720~16777216,268435456~8388608,281018368~16777216,314572800~8388608,348127232~12582912,364904448~8388608,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.428 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 8 diff [0~12582912,20971520~4194304,29360128~20971520,58720256~16777216,104857600~4194304,130023424~8388608,146800640~4194304,180355072~4194304,201326592~20971520,230686720~12582912,260046848~12582912,281018368~12582912,314572800~8388608,343932928~12582912,364904448~8388608,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.428 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~12582912,20971520~4194304,29360128~20971520,58720256~16777216,104857600~4194304,130023424~8388608,146800640~4194304,180355072~4194304,201326592~20971520,230686720~12582912,260046848~12582912,281018368~12582912,314572800~8388608,343932928~12582912,364904448~8388608,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.434 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~16777216,20971520~4194304,29360128~25165824,58720256~20971520,104857600~8388608,130023424~12582912,146800640~4194304,180355072~8388608,201326592~25165824,230686720~16777216,260046848~16777216,281018368~16777216,314572800~8388608,343932928~16777216,364904448~8388608,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.435 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 9 diff [0~12582912,20971520~29360128,58720256~20971520,96468992~12582912,130023424~8388608,146800640~4194304,155189248~4194304,180355072~4194304,201326592~20971520,230686720~12582912,260046848~12582912,276824064~25165824,314572800~8388608,343932928~12582912,364904448~8388608,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.435 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~12582912,20971520~29360128,58720256~20971520,96468992~12582912,130023424~8388608,146800640~4194304,155189248~4194304,180355072~4194304,201326592~20971520,230686720~12582912,260046848~12582912,276824064~25165824,314572800~8388608,343932928~12582912,364904448~8388608,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.442 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~16777216,20971520~33554432,58720256~25165824,96468992~16777216,130023424~12582912,146800640~4194304,155189248~4194304,180355072~8388608,201326592~25165824,230686720~16777216,260046848~46137344,314572800~8388608,343932928~16777216,364904448~8388608,385875968~8388608,402653184~4194304] 2024-03-07T03:48:34.442 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 10 diff [0~12582912,20971520~29360128,58720256~25165824,96468992~16777216,130023424~8388608,146800640~12582912,180355072~8388608,192937984~4194304,201326592~20971520,230686720~12582912,260046848~12582912,276824064~25165824,314572800~8388608,343932928~12582912,364904448~8388608,381681664~12582912,402653184~4194304] 2024-03-07T03:48:34.442 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~12582912,20971520~29360128,58720256~25165824,96468992~16777216,130023424~8388608,146800640~12582912,180355072~8388608,192937984~4194304,201326592~20971520,230686720~12582912,260046848~12582912,276824064~25165824,314572800~8388608,343932928~12582912,364904448~8388608,381681664~12582912,402653184~4194304] 2024-03-07T03:48:34.450 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~16777216,20971520~33554432,58720256~29360128,96468992~16777216,130023424~12582912,146800640~12582912,180355072~8388608,192937984~4194304,201326592~25165824,230686720~16777216,260046848~46137344,314572800~8388608,343932928~16777216,364904448~8388608,381681664~12582912,402653184~4194304] 2024-03-07T03:48:34.450 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 11 diff [0~50331648,54525952~33554432,96468992~16777216,130023424~12582912,146800640~12582912,180355072~8388608,192937984~4194304,201326592~20971520,230686720~12582912,260046848~12582912,276824064~25165824,314572800~8388608,343932928~12582912,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.450 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~50331648,54525952~33554432,96468992~16777216,130023424~12582912,146800640~12582912,180355072~8388608,192937984~4194304,201326592~20971520,230686720~12582912,260046848~12582912,276824064~25165824,314572800~8388608,343932928~12582912,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.460 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,130023424~12582912,146800640~12582912,180355072~8388608,192937984~4194304,201326592~25165824,230686720~16777216,260046848~46137344,314572800~8388608,343932928~29360128,381681664~25165824] 2024-03-07T03:48:34.460 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 12 diff [0~50331648,54525952~33554432,96468992~16777216,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~4194304,201326592~25165824,230686720~12582912,260046848~41943040,314572800~8388608,343932928~12582912,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.460 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~50331648,54525952~33554432,96468992~16777216,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~4194304,201326592~25165824,230686720~12582912,260046848~41943040,314572800~8388608,343932928~12582912,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.469 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~4194304,201326592~25165824,230686720~16777216,260046848~46137344,314572800~8388608,343932928~29360128,381681664~25165824] 2024-03-07T03:48:34.470 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 13 diff [0~50331648,54525952~33554432,96468992~16777216,130023424~12582912,146800640~41943040,192937984~4194304,201326592~25165824,230686720~12582912,260046848~41943040,314572800~8388608,343932928~12582912,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.470 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~50331648,54525952~33554432,96468992~16777216,130023424~12582912,146800640~41943040,192937984~4194304,201326592~25165824,230686720~12582912,260046848~41943040,314572800~8388608,343932928~12582912,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.481 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,130023424~12582912,146800640~50331648,201326592~25165824,230686720~16777216,260046848~46137344,314572800~8388608,343932928~29360128,381681664~25165824] 2024-03-07T03:48:34.481 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 14 diff [0~50331648,54525952~33554432,96468992~16777216,125829120~16777216,146800640~41943040,192937984~4194304,201326592~25165824,230686720~12582912,251658240~4194304,260046848~41943040,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.481 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~50331648,54525952~33554432,96468992~16777216,125829120~16777216,146800640~41943040,192937984~4194304,201326592~25165824,230686720~12582912,251658240~4194304,260046848~41943040,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.494 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,125829120~16777216,146800640~50331648,201326592~25165824,230686720~16777216,251658240~4194304,260046848~46137344,314572800~8388608,327155712~12582912,343932928~62914560] 2024-03-07T03:48:34.495 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 15 diff [0~50331648,54525952~33554432,96468992~12582912,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~12582912,251658240~4194304,260046848~41943040,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.495 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~50331648,54525952~33554432,96468992~12582912,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~12582912,251658240~4194304,260046848~41943040,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.508 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~16777216,251658240~4194304,260046848~46137344,314572800~8388608,327155712~12582912,343932928~62914560] 2024-03-07T03:48:34.508 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 16 diff [0~50331648,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~12582912,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.508 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~50331648,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~12582912,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.523 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~8388608,327155712~12582912,343932928~67108864] 2024-03-07T03:48:34.523 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 17 diff [0~50331648,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~16777216,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.523 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~50331648,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,146800640~50331648,201326592~25165824,230686720~16777216,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.538 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~8388608,327155712~12582912,343932928~67108864] 2024-03-07T03:48:34.538 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 18 diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.538 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.555 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:34.555 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to 19 diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.555 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.572 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:34.573 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 6 diff [29360128~4194304,104857600~4194304,146800640~4194304,180355072~8388608,209715200~4194304,234881024~12582912,268435456~4194304,314572800~4194304] 2024-03-07T03:48:34.573 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [29360128~4194304,104857600~4194304,146800640~4194304,180355072~8388608,209715200~4194304,234881024~12582912,268435456~4194304,314572800~4194304] 2024-03-07T03:48:34.576 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [29360128~8388608,104857600~8388608,146800640~4194304,180355072~8388608,209715200~4194304,234881024~12582912,268435456~8388608,314572800~8388608] 2024-03-07T03:48:34.576 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 7 diff [29360128~8388608,58720256~16777216,104857600~4194304,146800640~4194304,180355072~8388608,209715200~12582912,230686720~12582912,268435456~4194304,314572800~4194304,348127232~8388608,364904448~8388608] 2024-03-07T03:48:34.576 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [29360128~8388608,58720256~16777216,104857600~4194304,146800640~4194304,180355072~8388608,209715200~12582912,230686720~12582912,268435456~4194304,314572800~4194304,348127232~8388608,364904448~8388608] 2024-03-07T03:48:34.580 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [29360128~12582912,58720256~20971520,104857600~8388608,146800640~4194304,180355072~8388608,209715200~16777216,230686720~16777216,268435456~8388608,314572800~8388608,348127232~12582912,364904448~8388608] 2024-03-07T03:48:34.580 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 8 diff [0~4194304,20971520~4194304,29360128~20971520,58720256~16777216,104857600~4194304,130023424~8388608,146800640~4194304,180355072~8388608,209715200~12582912,230686720~12582912,260046848~12582912,314572800~8388608,343932928~12582912,364904448~8388608] 2024-03-07T03:48:34.580 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~4194304,20971520~4194304,29360128~20971520,58720256~16777216,104857600~4194304,130023424~8388608,146800640~4194304,180355072~8388608,209715200~12582912,230686720~12582912,260046848~12582912,314572800~8388608,343932928~12582912,364904448~8388608] 2024-03-07T03:48:34.585 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~4194304,20971520~4194304,29360128~25165824,58720256~20971520,104857600~8388608,130023424~12582912,146800640~4194304,180355072~8388608,209715200~16777216,230686720~16777216,260046848~16777216,314572800~8388608,343932928~16777216,364904448~8388608] 2024-03-07T03:48:34.585 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 9 diff [0~4194304,20971520~29360128,58720256~20971520,96468992~12582912,130023424~8388608,146800640~4194304,155189248~4194304,180355072~8388608,209715200~12582912,230686720~12582912,260046848~12582912,276824064~8388608,293601280~8388608,314572800~8388608,343932928~12582912,364904448~8388608] 2024-03-07T03:48:34.585 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~4194304,20971520~29360128,58720256~20971520,96468992~12582912,130023424~8388608,146800640~4194304,155189248~4194304,180355072~8388608,209715200~12582912,230686720~12582912,260046848~12582912,276824064~8388608,293601280~8388608,314572800~8388608,343932928~12582912,364904448~8388608] 2024-03-07T03:48:34.592 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~4194304,20971520~33554432,58720256~25165824,96468992~16777216,130023424~12582912,146800640~4194304,155189248~4194304,180355072~8388608,209715200~16777216,230686720~16777216,260046848~29360128,293601280~12582912,314572800~8388608,343932928~16777216,364904448~8388608] 2024-03-07T03:48:34.592 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 10 diff [0~4194304,20971520~29360128,58720256~25165824,96468992~16777216,130023424~8388608,146800640~12582912,180355072~8388608,192937984~4194304,209715200~12582912,230686720~12582912,260046848~12582912,276824064~8388608,293601280~8388608,314572800~8388608,343932928~12582912,364904448~8388608,381681664~4194304] 2024-03-07T03:48:34.592 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~4194304,20971520~29360128,58720256~25165824,96468992~16777216,130023424~8388608,146800640~12582912,180355072~8388608,192937984~4194304,209715200~12582912,230686720~12582912,260046848~12582912,276824064~8388608,293601280~8388608,314572800~8388608,343932928~12582912,364904448~8388608,381681664~4194304] 2024-03-07T03:48:34.600 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~4194304,20971520~33554432,58720256~29360128,96468992~16777216,130023424~12582912,146800640~12582912,180355072~8388608,192937984~4194304,209715200~16777216,230686720~16777216,260046848~29360128,293601280~12582912,314572800~8388608,343932928~16777216,364904448~8388608,381681664~4194304] 2024-03-07T03:48:34.600 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 11 diff [0~4194304,12582912~37748736,54525952~33554432,96468992~16777216,130023424~12582912,146800640~12582912,180355072~8388608,192937984~4194304,209715200~12582912,230686720~12582912,260046848~12582912,276824064~8388608,293601280~8388608,314572800~8388608,343932928~12582912,360710144~12582912,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.600 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~4194304,12582912~37748736,54525952~33554432,96468992~16777216,130023424~12582912,146800640~12582912,180355072~8388608,192937984~4194304,209715200~12582912,230686720~12582912,260046848~12582912,276824064~8388608,293601280~8388608,314572800~8388608,343932928~12582912,360710144~12582912,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.609 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~4194304,12582912~79691776,96468992~16777216,130023424~12582912,146800640~12582912,180355072~8388608,192937984~4194304,209715200~16777216,230686720~16777216,260046848~29360128,293601280~12582912,314572800~8388608,343932928~29360128,381681664~4194304,390070272~12582912] 2024-03-07T03:48:34.609 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 12 diff [0~8388608,12582912~37748736,54525952~33554432,96468992~16777216,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~4194304,209715200~16777216,230686720~12582912,260046848~25165824,289406976~12582912,314572800~8388608,343932928~12582912,360710144~12582912,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.609 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,96468992~16777216,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~4194304,209715200~16777216,230686720~12582912,260046848~25165824,289406976~12582912,314572800~8388608,343932928~12582912,360710144~12582912,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.618 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,130023424~12582912,146800640~12582912,167772160~8388608,180355072~8388608,192937984~4194304,209715200~16777216,230686720~16777216,260046848~46137344,314572800~8388608,343932928~29360128,381681664~4194304,390070272~12582912] 2024-03-07T03:48:34.618 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 13 diff [0~8388608,12582912~37748736,54525952~33554432,96468992~16777216,130023424~12582912,146800640~41943040,192937984~4194304,209715200~16777216,230686720~12582912,260046848~25165824,289406976~12582912,314572800~8388608,343932928~12582912,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.618 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,96468992~16777216,130023424~12582912,146800640~41943040,192937984~4194304,209715200~16777216,230686720~12582912,260046848~25165824,289406976~12582912,314572800~8388608,343932928~12582912,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.629 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,130023424~12582912,146800640~50331648,209715200~16777216,230686720~16777216,260046848~46137344,314572800~8388608,343932928~29360128,381681664~25165824] 2024-03-07T03:48:34.629 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 14 diff [0~8388608,12582912~37748736,54525952~33554432,96468992~16777216,125829120~16777216,146800640~41943040,192937984~4194304,209715200~16777216,230686720~12582912,251658240~4194304,260046848~25165824,289406976~12582912,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.629 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,96468992~16777216,125829120~16777216,146800640~41943040,192937984~4194304,209715200~16777216,230686720~12582912,251658240~4194304,260046848~25165824,289406976~12582912,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.642 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,125829120~16777216,146800640~50331648,209715200~16777216,230686720~16777216,251658240~4194304,260046848~46137344,314572800~8388608,327155712~12582912,343932928~62914560] 2024-03-07T03:48:34.642 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 15 diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,117440512~4194304,125829120~16777216,146800640~50331648,205520896~20971520,230686720~12582912,251658240~4194304,260046848~41943040,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.642 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,117440512~4194304,125829120~16777216,146800640~50331648,205520896~20971520,230686720~12582912,251658240~4194304,260046848~41943040,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.655 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~16777216,117440512~4194304,125829120~16777216,146800640~50331648,205520896~20971520,230686720~16777216,251658240~4194304,260046848~46137344,314572800~8388608,327155712~12582912,343932928~62914560] 2024-03-07T03:48:34.655 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 16 diff [0~8388608,12582912~37748736,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,146800640~50331648,205520896~20971520,230686720~12582912,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.655 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,146800640~50331648,205520896~20971520,230686720~12582912,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.668 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~54525952,314572800~8388608,327155712~12582912,343932928~67108864] 2024-03-07T03:48:34.668 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 17 diff [0~50331648,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,146800640~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.668 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~50331648,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,146800640~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.683 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~8388608,327155712~12582912,343932928~67108864] 2024-03-07T03:48:34.683 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 18 diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.683 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.698 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:34.699 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to 19 diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.699 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.715 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:34.715 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 7 diff [33554432~4194304,58720256~16777216,213909504~8388608,230686720~8388608,348127232~8388608,364904448~8388608] 2024-03-07T03:48:34.716 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [33554432~4194304,58720256~16777216,213909504~8388608,230686720~8388608,348127232~8388608,364904448~8388608] 2024-03-07T03:48:34.719 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [33554432~8388608,58720256~20971520,213909504~12582912,230686720~12582912,348127232~12582912,364904448~8388608] 2024-03-07T03:48:34.719 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 8 diff [0~4194304,20971520~4194304,33554432~16777216,58720256~16777216,130023424~8388608,213909504~8388608,230686720~8388608,260046848~8388608,314572800~8388608,343932928~12582912,364904448~8388608] 2024-03-07T03:48:34.719 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~4194304,20971520~4194304,33554432~16777216,58720256~16777216,130023424~8388608,213909504~8388608,230686720~8388608,260046848~8388608,314572800~8388608,343932928~12582912,364904448~8388608] 2024-03-07T03:48:34.725 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~4194304,20971520~4194304,33554432~20971520,58720256~20971520,130023424~12582912,213909504~12582912,230686720~12582912,260046848~12582912,314572800~8388608,343932928~16777216,364904448~8388608] 2024-03-07T03:48:34.725 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 9 diff [0~4194304,20971520~29360128,58720256~20971520,96468992~8388608,130023424~8388608,155189248~4194304,213909504~8388608,230686720~12582912,260046848~8388608,276824064~8388608,293601280~8388608,314572800~8388608,343932928~12582912,364904448~8388608] 2024-03-07T03:48:34.725 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~4194304,20971520~29360128,58720256~20971520,96468992~8388608,130023424~8388608,155189248~4194304,213909504~8388608,230686720~12582912,260046848~8388608,276824064~8388608,293601280~8388608,314572800~8388608,343932928~12582912,364904448~8388608] 2024-03-07T03:48:34.731 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~4194304,20971520~33554432,58720256~25165824,96468992~12582912,130023424~12582912,155189248~4194304,213909504~12582912,230686720~16777216,260046848~12582912,276824064~12582912,293601280~12582912,314572800~8388608,343932928~16777216,364904448~8388608] 2024-03-07T03:48:34.731 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 10 diff [0~4194304,20971520~29360128,58720256~25165824,96468992~12582912,130023424~8388608,150994944~8388608,180355072~8388608,192937984~4194304,213909504~8388608,230686720~12582912,260046848~8388608,276824064~8388608,293601280~8388608,314572800~8388608,343932928~12582912,364904448~8388608,381681664~4194304] 2024-03-07T03:48:34.731 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~4194304,20971520~29360128,58720256~25165824,96468992~12582912,130023424~8388608,150994944~8388608,180355072~8388608,192937984~4194304,213909504~8388608,230686720~12582912,260046848~8388608,276824064~8388608,293601280~8388608,314572800~8388608,343932928~12582912,364904448~8388608,381681664~4194304] 2024-03-07T03:48:34.737 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~4194304,20971520~33554432,58720256~29360128,96468992~12582912,130023424~12582912,150994944~8388608,180355072~8388608,192937984~4194304,213909504~12582912,230686720~16777216,260046848~12582912,276824064~12582912,293601280~12582912,314572800~8388608,343932928~16777216,364904448~8388608,381681664~4194304] 2024-03-07T03:48:34.738 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 11 diff [0~4194304,12582912~37748736,54525952~33554432,96468992~12582912,130023424~12582912,150994944~8388608,180355072~8388608,192937984~4194304,213909504~8388608,230686720~12582912,260046848~8388608,276824064~8388608,293601280~8388608,314572800~8388608,343932928~12582912,360710144~12582912,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.738 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~4194304,12582912~37748736,54525952~33554432,96468992~12582912,130023424~12582912,150994944~8388608,180355072~8388608,192937984~4194304,213909504~8388608,230686720~12582912,260046848~8388608,276824064~8388608,293601280~8388608,314572800~8388608,343932928~12582912,360710144~12582912,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.746 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~4194304,12582912~79691776,96468992~12582912,130023424~12582912,150994944~8388608,180355072~8388608,192937984~4194304,213909504~12582912,230686720~16777216,260046848~12582912,276824064~12582912,293601280~12582912,314572800~8388608,343932928~29360128,381681664~4194304,390070272~12582912] 2024-03-07T03:48:34.746 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 12 diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,130023424~12582912,150994944~8388608,167772160~8388608,180355072~8388608,192937984~4194304,213909504~12582912,230686720~12582912,260046848~25165824,289406976~12582912,314572800~8388608,343932928~12582912,360710144~12582912,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.746 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,130023424~12582912,150994944~8388608,167772160~8388608,180355072~8388608,192937984~4194304,213909504~12582912,230686720~12582912,260046848~25165824,289406976~12582912,314572800~8388608,343932928~12582912,360710144~12582912,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.755 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~12582912,130023424~12582912,150994944~8388608,167772160~8388608,180355072~8388608,192937984~4194304,213909504~12582912,230686720~16777216,260046848~46137344,314572800~8388608,343932928~29360128,381681664~4194304,390070272~12582912] 2024-03-07T03:48:34.755 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 13 diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,130023424~12582912,150994944~37748736,192937984~4194304,213909504~12582912,230686720~12582912,260046848~25165824,289406976~12582912,314572800~8388608,343932928~12582912,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.755 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,130023424~12582912,150994944~37748736,192937984~4194304,213909504~12582912,230686720~12582912,260046848~25165824,289406976~12582912,314572800~8388608,343932928~12582912,360710144~12582912,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.765 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~12582912,130023424~12582912,150994944~46137344,213909504~12582912,230686720~16777216,260046848~46137344,314572800~8388608,343932928~29360128,381681664~25165824] 2024-03-07T03:48:34.765 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 14 diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,125829120~16777216,150994944~37748736,192937984~4194304,209715200~16777216,230686720~12582912,251658240~4194304,260046848~25165824,289406976~12582912,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.765 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,125829120~16777216,150994944~37748736,192937984~4194304,209715200~16777216,230686720~12582912,251658240~4194304,260046848~25165824,289406976~12582912,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.775 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~12582912,125829120~16777216,150994944~46137344,209715200~16777216,230686720~16777216,251658240~4194304,260046848~46137344,314572800~8388608,327155712~12582912,343932928~62914560] 2024-03-07T03:48:34.775 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 15 diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,117440512~4194304,125829120~16777216,150994944~46137344,205520896~20971520,230686720~12582912,251658240~4194304,260046848~41943040,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.775 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,117440512~4194304,125829120~16777216,150994944~46137344,205520896~20971520,230686720~12582912,251658240~4194304,260046848~41943040,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.787 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~12582912,117440512~4194304,125829120~16777216,150994944~46137344,205520896~20971520,230686720~16777216,251658240~4194304,260046848~46137344,314572800~8388608,327155712~12582912,343932928~62914560] 2024-03-07T03:48:34.787 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 16 diff [0~8388608,12582912~37748736,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,150994944~46137344,205520896~20971520,230686720~12582912,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.787 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,150994944~46137344,205520896~20971520,230686720~12582912,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.800 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~54525952,314572800~8388608,327155712~12582912,343932928~67108864] 2024-03-07T03:48:34.800 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 17 diff [0~50331648,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,150994944~46137344,205520896~20971520,230686720~16777216,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.800 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~50331648,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,150994944~46137344,205520896~20971520,230686720~16777216,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.813 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~8388608,327155712~12582912,343932928~67108864] 2024-03-07T03:48:34.813 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 18 diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.813 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:34 smithi145 bash[79878]: cluster 2024-03-07T03:48:33.510232+0000 mon.a (mon.0) 1929 : cluster 0 osdmap e1076: 8 total, 8 up, 8 in 2024-03-07T03:48:34.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:34 smithi145 bash[79878]: cluster 2024-03-07T03:48:33.651074+0000 mgr.y (mgr.34107) 1101 : cluster 0 pgmap v1802: 105 pgs: 105 active+clean; 806 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 244 KiB/s rd, 46 MiB/s wr, 380 op/s 2024-03-07T03:48:34.817 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:34 smithi145 bash[82329]: cluster 2024-03-07T03:48:33.510232+0000 mon.a (mon.0) 1929 : cluster 0 osdmap e1076: 8 total, 8 up, 8 in 2024-03-07T03:48:34.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:34 smithi145 bash[82329]: cluster 2024-03-07T03:48:33.651074+0000 mgr.y (mgr.34107) 1101 : cluster 0 pgmap v1802: 105 pgs: 105 active+clean; 806 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 244 KiB/s rd, 46 MiB/s wr, 380 op/s 2024-03-07T03:48:34.828 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:34.828 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to 19 diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.828 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:34 smithi189 bash[66969]: cluster 2024-03-07T03:48:33.510232+0000 mon.a (mon.0) 1929 : cluster 0 osdmap e1076: 8 total, 8 up, 8 in 2024-03-07T03:48:34.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:34 smithi189 bash[66969]: cluster 2024-03-07T03:48:33.651074+0000 mgr.y (mgr.34107) 1101 : cluster 0 pgmap v1802: 105 pgs: 105 active+clean; 806 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 244 KiB/s rd, 46 MiB/s wr, 380 op/s 2024-03-07T03:48:34.844 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:34.844 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 8 diff [0~4194304,20971520~4194304,37748736~12582912,67108864~4194304,130023424~8388608,260046848~8388608,314572800~8388608,343932928~8388608] 2024-03-07T03:48:34.844 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~4194304,20971520~4194304,37748736~12582912,67108864~4194304,130023424~8388608,260046848~8388608,314572800~8388608,343932928~8388608] 2024-03-07T03:48:34.848 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~4194304,20971520~4194304,37748736~16777216,67108864~8388608,130023424~12582912,260046848~12582912,314572800~8388608,343932928~8388608] 2024-03-07T03:48:34.848 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 9 diff [0~4194304,20971520~12582912,37748736~12582912,58720256~4194304,67108864~12582912,96468992~8388608,130023424~8388608,155189248~4194304,239075328~8388608,260046848~8388608,276824064~8388608,293601280~8388608,314572800~8388608,343932928~8388608] 2024-03-07T03:48:34.848 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~4194304,20971520~12582912,37748736~12582912,58720256~4194304,67108864~12582912,96468992~8388608,130023424~8388608,155189248~4194304,239075328~8388608,260046848~8388608,276824064~8388608,293601280~8388608,314572800~8388608,343932928~8388608] 2024-03-07T03:48:34.852 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~4194304,20971520~33554432,58720256~25165824,96468992~12582912,130023424~12582912,155189248~4194304,239075328~8388608,260046848~12582912,276824064~12582912,293601280~12582912,314572800~8388608,343932928~8388608] 2024-03-07T03:48:34.852 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 10 diff [0~4194304,20971520~12582912,37748736~12582912,58720256~4194304,67108864~16777216,96468992~12582912,130023424~8388608,150994944~8388608,180355072~8388608,192937984~4194304,234881024~8388608,260046848~8388608,276824064~8388608,293601280~8388608,314572800~8388608,343932928~8388608,381681664~4194304] 2024-03-07T03:48:34.853 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~4194304,20971520~12582912,37748736~12582912,58720256~4194304,67108864~16777216,96468992~12582912,130023424~8388608,150994944~8388608,180355072~8388608,192937984~4194304,234881024~8388608,260046848~8388608,276824064~8388608,293601280~8388608,314572800~8388608,343932928~8388608,381681664~4194304] 2024-03-07T03:48:34.858 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~4194304,20971520~33554432,58720256~29360128,96468992~12582912,130023424~12582912,150994944~8388608,180355072~8388608,192937984~4194304,234881024~12582912,260046848~12582912,276824064~12582912,293601280~12582912,314572800~8388608,343932928~8388608,381681664~4194304] 2024-03-07T03:48:34.858 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 11 diff [0~4194304,12582912~20971520,37748736~12582912,54525952~8388608,67108864~20971520,96468992~12582912,130023424~12582912,150994944~8388608,180355072~8388608,192937984~4194304,234881024~8388608,260046848~8388608,276824064~8388608,293601280~8388608,314572800~8388608,343932928~8388608,360710144~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.858 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~4194304,12582912~20971520,37748736~12582912,54525952~8388608,67108864~20971520,96468992~12582912,130023424~12582912,150994944~8388608,180355072~8388608,192937984~4194304,234881024~8388608,260046848~8388608,276824064~8388608,293601280~8388608,314572800~8388608,343932928~8388608,360710144~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.865 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~4194304,12582912~79691776,96468992~12582912,130023424~12582912,150994944~8388608,180355072~8388608,192937984~4194304,234881024~12582912,260046848~12582912,276824064~12582912,293601280~12582912,314572800~8388608,343932928~8388608,360710144~8388608,381681664~4194304,390070272~12582912] 2024-03-07T03:48:34.865 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 12 diff [0~8388608,12582912~37748736,54525952~8388608,67108864~20971520,96468992~12582912,130023424~12582912,150994944~8388608,167772160~8388608,180355072~8388608,192937984~4194304,222298112~4194304,234881024~8388608,260046848~25165824,289406976~12582912,314572800~8388608,343932928~8388608,360710144~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.865 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~8388608,67108864~20971520,96468992~12582912,130023424~12582912,150994944~8388608,167772160~8388608,180355072~8388608,192937984~4194304,222298112~4194304,234881024~8388608,260046848~25165824,289406976~12582912,314572800~8388608,343932928~8388608,360710144~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.873 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~12582912,130023424~12582912,150994944~8388608,167772160~8388608,180355072~8388608,192937984~4194304,222298112~4194304,234881024~12582912,260046848~46137344,314572800~8388608,343932928~8388608,360710144~12582912,381681664~4194304,390070272~12582912] 2024-03-07T03:48:34.873 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 13 diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,130023424~12582912,150994944~37748736,192937984~4194304,222298112~4194304,230686720~12582912,260046848~25165824,289406976~12582912,314572800~8388608,343932928~8388608,360710144~8388608,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.873 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,130023424~12582912,150994944~37748736,192937984~4194304,222298112~4194304,230686720~12582912,260046848~25165824,289406976~12582912,314572800~8388608,343932928~8388608,360710144~8388608,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.882 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~12582912,130023424~12582912,150994944~46137344,222298112~4194304,230686720~16777216,260046848~46137344,314572800~8388608,343932928~8388608,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.882 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 14 diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,125829120~16777216,150994944~37748736,192937984~4194304,209715200~4194304,218103808~4194304,230686720~12582912,251658240~4194304,260046848~25165824,289406976~12582912,314572800~8388608,327155712~8388608,343932928~8388608,356515840~12582912,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.882 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,125829120~16777216,150994944~37748736,192937984~4194304,209715200~4194304,218103808~4194304,230686720~12582912,251658240~4194304,260046848~25165824,289406976~12582912,314572800~8388608,327155712~8388608,343932928~8388608,356515840~12582912,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.891 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~12582912,125829120~16777216,150994944~46137344,209715200~16777216,230686720~16777216,251658240~4194304,260046848~46137344,314572800~8388608,327155712~12582912,343932928~62914560] 2024-03-07T03:48:34.891 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 15 diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,117440512~4194304,125829120~16777216,150994944~46137344,205520896~8388608,218103808~4194304,230686720~12582912,251658240~4194304,260046848~41943040,314572800~8388608,327155712~8388608,343932928~8388608,356515840~12582912,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.891 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,96468992~12582912,117440512~4194304,125829120~16777216,150994944~46137344,205520896~8388608,218103808~4194304,230686720~12582912,251658240~4194304,260046848~41943040,314572800~8388608,327155712~8388608,343932928~8388608,356515840~12582912,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.902 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~92274688,96468992~12582912,117440512~4194304,125829120~16777216,150994944~46137344,205520896~20971520,230686720~16777216,251658240~4194304,260046848~46137344,314572800~8388608,327155712~12582912,343932928~62914560] 2024-03-07T03:48:34.902 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 16 diff [0~8388608,12582912~37748736,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,150994944~46137344,205520896~8388608,218103808~4194304,230686720~12582912,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.902 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~8388608,12582912~37748736,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,150994944~46137344,205520896~8388608,218103808~4194304,230686720~12582912,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.914 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~54525952,314572800~8388608,327155712~12582912,343932928~67108864] 2024-03-07T03:48:34.914 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 17 diff [0~50331648,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,150994944~46137344,205520896~20971520,230686720~16777216,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.914 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~50331648,54525952~33554432,92274688~20971520,117440512~4194304,125829120~16777216,150994944~46137344,205520896~20971520,230686720~16777216,251658240~50331648,314572800~8388608,327155712~8388608,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.926 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~8388608,327155712~12582912,343932928~67108864] 2024-03-07T03:48:34.926 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 18 diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.927 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.940 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:34.940 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to 19 diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.941 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:34.955 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:34.955 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 9 diff [25165824~8388608,58720256~4194304,71303168~8388608,96468992~8388608,155189248~4194304,239075328~8388608,276824064~8388608,293601280~8388608,314572800~4194304] 2024-03-07T03:48:34.955 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [25165824~8388608,58720256~4194304,71303168~8388608,96468992~8388608,155189248~4194304,239075328~8388608,276824064~8388608,293601280~8388608,314572800~4194304] 2024-03-07T03:48:34.958 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [25165824~12582912,58720256~8388608,71303168~12582912,96468992~12582912,155189248~4194304,239075328~8388608,276824064~12582912,293601280~12582912,314572800~4194304] 2024-03-07T03:48:34.958 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 10 diff [25165824~8388608,46137344~4194304,58720256~4194304,71303168~12582912,96468992~12582912,150994944~8388608,180355072~8388608,192937984~4194304,234881024~8388608,276824064~8388608,293601280~8388608,314572800~4194304,348127232~4194304,381681664~4194304] 2024-03-07T03:48:34.959 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [25165824~8388608,46137344~4194304,58720256~4194304,71303168~12582912,96468992~12582912,150994944~8388608,180355072~8388608,192937984~4194304,234881024~8388608,276824064~8388608,293601280~8388608,314572800~4194304,348127232~4194304,381681664~4194304] 2024-03-07T03:48:34.963 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [25165824~12582912,46137344~8388608,58720256~8388608,71303168~16777216,96468992~12582912,150994944~8388608,180355072~8388608,192937984~4194304,234881024~12582912,276824064~12582912,293601280~12582912,314572800~4194304,348127232~4194304,381681664~4194304] 2024-03-07T03:48:34.963 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 11 diff [12582912~20971520,46137344~4194304,54525952~8388608,71303168~16777216,96468992~12582912,130023424~8388608,150994944~8388608,180355072~8388608,192937984~4194304,234881024~8388608,268435456~4194304,276824064~8388608,293601280~8388608,314572800~4194304,348127232~4194304,360710144~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.963 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [12582912~20971520,46137344~4194304,54525952~8388608,71303168~16777216,96468992~12582912,130023424~8388608,150994944~8388608,180355072~8388608,192937984~4194304,234881024~8388608,268435456~4194304,276824064~8388608,293601280~8388608,314572800~4194304,348127232~4194304,360710144~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.969 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [12582912~25165824,46137344~20971520,71303168~20971520,96468992~12582912,130023424~8388608,150994944~8388608,180355072~8388608,192937984~4194304,234881024~12582912,268435456~4194304,276824064~12582912,293601280~12582912,314572800~4194304,348127232~4194304,360710144~8388608,381681664~4194304,390070272~12582912] 2024-03-07T03:48:34.969 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 12 diff [4194304~4194304,12582912~29360128,46137344~4194304,54525952~8388608,71303168~16777216,96468992~12582912,130023424~8388608,150994944~8388608,167772160~8388608,180355072~8388608,192937984~4194304,222298112~4194304,234881024~8388608,260046848~25165824,289406976~12582912,314572800~4194304,348127232~4194304,360710144~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.969 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~29360128,46137344~4194304,54525952~8388608,71303168~16777216,96468992~12582912,130023424~8388608,150994944~8388608,167772160~8388608,180355072~8388608,192937984~4194304,222298112~4194304,234881024~8388608,260046848~25165824,289406976~12582912,314572800~4194304,348127232~4194304,360710144~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:34.976 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~62914560,71303168~20971520,96468992~12582912,130023424~8388608,150994944~8388608,167772160~8388608,180355072~8388608,192937984~4194304,222298112~4194304,234881024~12582912,260046848~46137344,314572800~4194304,348127232~4194304,360710144~12582912,381681664~4194304,390070272~12582912] 2024-03-07T03:48:34.976 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 13 diff [4194304~4194304,12582912~29360128,46137344~4194304,54525952~12582912,71303168~16777216,96468992~12582912,130023424~8388608,150994944~37748736,192937984~4194304,222298112~4194304,230686720~12582912,260046848~25165824,289406976~12582912,314572800~4194304,348127232~4194304,360710144~8388608,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.976 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~29360128,46137344~4194304,54525952~12582912,71303168~16777216,96468992~12582912,130023424~8388608,150994944~37748736,192937984~4194304,222298112~4194304,230686720~12582912,260046848~25165824,289406976~12582912,314572800~4194304,348127232~4194304,360710144~8388608,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.985 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~88080384,96468992~12582912,130023424~8388608,150994944~46137344,222298112~4194304,230686720~16777216,260046848~46137344,314572800~4194304,348127232~4194304,360710144~12582912,381681664~25165824] 2024-03-07T03:48:34.985 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 14 diff [4194304~4194304,12582912~29360128,46137344~4194304,54525952~12582912,71303168~16777216,96468992~12582912,125829120~12582912,150994944~37748736,192937984~4194304,209715200~4194304,218103808~4194304,230686720~12582912,251658240~4194304,260046848~25165824,289406976~12582912,314572800~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.985 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~29360128,46137344~4194304,54525952~12582912,71303168~16777216,96468992~12582912,125829120~12582912,150994944~37748736,192937984~4194304,209715200~4194304,218103808~4194304,230686720~12582912,251658240~4194304,260046848~25165824,289406976~12582912,314572800~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.994 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~88080384,96468992~12582912,125829120~12582912,150994944~46137344,209715200~16777216,230686720~16777216,251658240~4194304,260046848~46137344,314572800~4194304,327155712~12582912,348127232~58720256] 2024-03-07T03:48:34.994 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 15 diff [4194304~4194304,12582912~29360128,46137344~4194304,54525952~12582912,71303168~16777216,96468992~12582912,117440512~4194304,125829120~12582912,150994944~46137344,205520896~8388608,218103808~4194304,230686720~12582912,251658240~4194304,260046848~41943040,314572800~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:34.994 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~29360128,46137344~4194304,54525952~12582912,71303168~16777216,96468992~12582912,117440512~4194304,125829120~12582912,150994944~46137344,205520896~8388608,218103808~4194304,230686720~12582912,251658240~4194304,260046848~41943040,314572800~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:35.003 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~88080384,96468992~12582912,117440512~4194304,125829120~12582912,150994944~46137344,205520896~20971520,230686720~16777216,251658240~4194304,260046848~46137344,314572800~4194304,327155712~12582912,348127232~58720256] 2024-03-07T03:48:35.003 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 16 diff [4194304~4194304,12582912~29360128,46137344~4194304,54525952~12582912,71303168~16777216,92274688~20971520,117440512~4194304,125829120~12582912,150994944~46137344,205520896~8388608,218103808~4194304,230686720~12582912,251658240~50331648,314572800~4194304,327155712~8388608,348127232~29360128,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.003 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~29360128,46137344~4194304,54525952~12582912,71303168~16777216,92274688~20971520,117440512~4194304,125829120~12582912,150994944~46137344,205520896~8388608,218103808~4194304,230686720~12582912,251658240~50331648,314572800~4194304,327155712~8388608,348127232~29360128,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.015 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~54525952,314572800~4194304,327155712~12582912,348127232~62914560] 2024-03-07T03:48:35.015 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 17 diff [4194304~37748736,46137344~4194304,54525952~12582912,71303168~41943040,117440512~4194304,125829120~12582912,150994944~46137344,205520896~20971520,230686720~16777216,251658240~50331648,314572800~4194304,327155712~8388608,348127232~29360128,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.016 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~37748736,46137344~4194304,54525952~12582912,71303168~41943040,117440512~4194304,125829120~12582912,150994944~46137344,205520896~20971520,230686720~16777216,251658240~50331648,314572800~4194304,327155712~8388608,348127232~29360128,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.027 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~75497472,314572800~4194304,327155712~12582912,348127232~62914560] 2024-03-07T03:48:35.027 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 18 diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~4194304,322961408~12582912,343932928~54525952,402653184~8388608] 2024-03-07T03:48:35.027 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~4194304,322961408~12582912,343932928~54525952,402653184~8388608] 2024-03-07T03:48:35.039 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~75497472,314572800~4194304,322961408~16777216,343932928~67108864] 2024-03-07T03:48:35.040 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to 19 diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.040 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.052 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:35.053 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 10 diff [46137344~4194304,79691776~4194304,104857600~4194304,150994944~8388608,180355072~8388608,192937984~4194304,234881024~8388608,348127232~4194304,381681664~4194304] 2024-03-07T03:48:35.053 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [46137344~4194304,79691776~4194304,104857600~4194304,150994944~8388608,180355072~8388608,192937984~4194304,234881024~8388608,348127232~4194304,381681664~4194304] 2024-03-07T03:48:35.057 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [46137344~8388608,79691776~8388608,104857600~4194304,150994944~8388608,180355072~8388608,192937984~4194304,234881024~12582912,348127232~4194304,381681664~4194304] 2024-03-07T03:48:35.057 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 11 diff [12582912~12582912,46137344~4194304,54525952~8388608,71303168~4194304,79691776~8388608,104857600~4194304,130023424~8388608,150994944~8388608,180355072~8388608,192937984~4194304,234881024~8388608,268435456~4194304,348127232~4194304,360710144~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:35.057 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [12582912~12582912,46137344~4194304,54525952~8388608,71303168~4194304,79691776~8388608,104857600~4194304,130023424~8388608,150994944~8388608,180355072~8388608,192937984~4194304,234881024~8388608,268435456~4194304,348127232~4194304,360710144~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:35.062 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [12582912~12582912,46137344~20971520,71303168~20971520,104857600~4194304,130023424~8388608,150994944~8388608,180355072~8388608,192937984~4194304,234881024~12582912,268435456~4194304,348127232~4194304,360710144~8388608,381681664~4194304,390070272~12582912] 2024-03-07T03:48:35.062 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 12 diff [4194304~4194304,12582912~12582912,33554432~8388608,46137344~4194304,54525952~8388608,71303168~16777216,104857600~4194304,130023424~8388608,150994944~8388608,167772160~8388608,180355072~8388608,192937984~4194304,222298112~4194304,234881024~8388608,260046848~16777216,289406976~4194304,348127232~4194304,360710144~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:35.062 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~12582912,33554432~8388608,46137344~4194304,54525952~8388608,71303168~16777216,104857600~4194304,130023424~8388608,150994944~8388608,167772160~8388608,180355072~8388608,192937984~4194304,222298112~4194304,234881024~8388608,260046848~16777216,289406976~4194304,348127232~4194304,360710144~8388608,381681664~4194304,390070272~8388608] 2024-03-07T03:48:35.067 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~20971520,33554432~33554432,71303168~20971520,104857600~4194304,130023424~8388608,150994944~8388608,167772160~8388608,180355072~8388608,192937984~4194304,222298112~4194304,234881024~12582912,260046848~16777216,289406976~4194304,348127232~4194304,360710144~12582912,381681664~4194304,390070272~12582912] 2024-03-07T03:48:35.068 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 13 diff [4194304~4194304,12582912~12582912,33554432~8388608,46137344~4194304,54525952~12582912,71303168~16777216,104857600~4194304,130023424~8388608,150994944~37748736,192937984~4194304,222298112~4194304,230686720~12582912,260046848~16777216,289406976~4194304,297795584~4194304,348127232~4194304,360710144~8388608,381681664~16777216,402653184~4194304] 2024-03-07T03:48:35.068 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~12582912,33554432~8388608,46137344~4194304,54525952~12582912,71303168~16777216,104857600~4194304,130023424~8388608,150994944~37748736,192937984~4194304,222298112~4194304,230686720~12582912,260046848~16777216,289406976~4194304,297795584~4194304,348127232~4194304,360710144~8388608,381681664~16777216,402653184~4194304] 2024-03-07T03:48:35.074 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~20971520,33554432~58720256,104857600~4194304,130023424~8388608,150994944~46137344,222298112~4194304,230686720~16777216,260046848~16777216,289406976~4194304,297795584~8388608,348127232~4194304,360710144~12582912,381681664~25165824] 2024-03-07T03:48:35.074 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 14 diff [4194304~4194304,12582912~12582912,33554432~8388608,46137344~4194304,54525952~12582912,71303168~16777216,104857600~4194304,125829120~12582912,150994944~37748736,192937984~4194304,209715200~4194304,218103808~4194304,230686720~12582912,251658240~4194304,260046848~16777216,289406976~4194304,297795584~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:35.074 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~12582912,33554432~8388608,46137344~4194304,54525952~12582912,71303168~16777216,104857600~4194304,125829120~12582912,150994944~37748736,192937984~4194304,209715200~4194304,218103808~4194304,230686720~12582912,251658240~4194304,260046848~16777216,289406976~4194304,297795584~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:35.082 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~20971520,33554432~58720256,104857600~4194304,125829120~12582912,150994944~46137344,209715200~16777216,230686720~16777216,251658240~4194304,260046848~16777216,289406976~4194304,297795584~8388608,327155712~12582912,348127232~58720256] 2024-03-07T03:48:35.082 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 15 diff [4194304~4194304,12582912~12582912,33554432~8388608,46137344~4194304,54525952~12582912,71303168~16777216,100663296~8388608,117440512~4194304,125829120~12582912,150994944~46137344,205520896~8388608,218103808~4194304,230686720~12582912,251658240~4194304,260046848~16777216,281018368~12582912,297795584~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:35.082 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~12582912,33554432~8388608,46137344~4194304,54525952~12582912,71303168~16777216,100663296~8388608,117440512~4194304,125829120~12582912,150994944~46137344,205520896~8388608,218103808~4194304,230686720~12582912,251658240~4194304,260046848~16777216,281018368~12582912,297795584~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:35.091 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~20971520,33554432~58720256,100663296~8388608,117440512~4194304,125829120~12582912,150994944~46137344,205520896~20971520,230686720~16777216,251658240~4194304,260046848~16777216,281018368~12582912,297795584~8388608,327155712~12582912,348127232~58720256] 2024-03-07T03:48:35.091 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 16 diff [4194304~4194304,12582912~12582912,33554432~8388608,46137344~4194304,54525952~12582912,71303168~16777216,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,150994944~46137344,205520896~8388608,218103808~4194304,230686720~12582912,251658240~25165824,281018368~12582912,297795584~4194304,327155712~8388608,348127232~29360128,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.091 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~12582912,33554432~8388608,46137344~4194304,54525952~12582912,71303168~16777216,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,150994944~46137344,205520896~8388608,218103808~4194304,230686720~12582912,251658240~25165824,281018368~12582912,297795584~4194304,327155712~8388608,348127232~29360128,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.101 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~20971520,33554432~62914560,100663296~20971520,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~8388608,327155712~12582912,348127232~62914560] 2024-03-07T03:48:35.101 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 17 diff [4194304~25165824,33554432~8388608,46137344~4194304,54525952~12582912,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~46137344,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,327155712~8388608,348127232~29360128,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.101 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~4194304,54525952~12582912,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~46137344,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,327155712~8388608,348127232~29360128,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.113 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~62914560,100663296~20971520,125829120~12582912,150994944~50331648,205520896~20971520,230686720~46137344,281018368~12582912,297795584~8388608,327155712~12582912,348127232~62914560] 2024-03-07T03:48:35.113 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 18 diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,322961408~12582912,343932928~54525952,402653184~8388608] 2024-03-07T03:48:35.113 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,322961408~12582912,343932928~54525952,402653184~8388608] 2024-03-07T03:48:35.125 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~62914560,100663296~20971520,125829120~12582912,150994944~50331648,205520896~20971520,230686720~46137344,281018368~12582912,297795584~8388608,322961408~16777216,343932928~67108864] 2024-03-07T03:48:35.125 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to 19 diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.125 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.138 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~62914560,100663296~20971520,125829120~12582912,150994944~50331648,205520896~20971520,230686720~46137344,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:35.138 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 11 diff [12582912~12582912,54525952~8388608,71303168~4194304,83886080~8388608,130023424~8388608,268435456~4194304,360710144~8388608,390070272~8388608] 2024-03-07T03:48:35.139 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [12582912~12582912,54525952~8388608,71303168~4194304,83886080~8388608,130023424~8388608,268435456~4194304,360710144~8388608,390070272~8388608] 2024-03-07T03:48:35.142 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [12582912~12582912,54525952~12582912,71303168~8388608,83886080~8388608,130023424~8388608,268435456~4194304,360710144~8388608,390070272~12582912] 2024-03-07T03:48:35.142 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 12 diff [4194304~4194304,12582912~12582912,33554432~8388608,54525952~8388608,71303168~20971520,130023424~8388608,167772160~8388608,222298112~4194304,239075328~8388608,260046848~16777216,289406976~4194304,360710144~8388608,390070272~8388608] 2024-03-07T03:48:35.142 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~12582912,33554432~8388608,54525952~8388608,71303168~20971520,130023424~8388608,167772160~8388608,222298112~4194304,239075328~8388608,260046848~16777216,289406976~4194304,360710144~8388608,390070272~8388608] 2024-03-07T03:48:35.147 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~20971520,33554432~12582912,54525952~12582912,71303168~20971520,130023424~8388608,167772160~8388608,222298112~4194304,239075328~8388608,260046848~16777216,289406976~4194304,360710144~12582912,390070272~12582912] 2024-03-07T03:48:35.147 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 13 diff [4194304~4194304,12582912~12582912,33554432~8388608,54525952~12582912,71303168~20971520,130023424~8388608,159383552~29360128,222298112~4194304,230686720~4194304,239075328~8388608,260046848~16777216,289406976~4194304,297795584~4194304,360710144~8388608,381681664~16777216,402653184~4194304] 2024-03-07T03:48:35.147 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~12582912,33554432~8388608,54525952~12582912,71303168~20971520,130023424~8388608,159383552~29360128,222298112~4194304,230686720~4194304,239075328~8388608,260046848~16777216,289406976~4194304,297795584~4194304,360710144~8388608,381681664~16777216,402653184~4194304] 2024-03-07T03:48:35.153 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~20971520,33554432~12582912,54525952~37748736,130023424~8388608,159383552~33554432,222298112~4194304,230686720~4194304,239075328~8388608,260046848~16777216,289406976~4194304,297795584~8388608,360710144~12582912,381681664~25165824] 2024-03-07T03:48:35.153 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 14 diff [4194304~4194304,12582912~12582912,33554432~8388608,54525952~12582912,71303168~20971520,125829120~12582912,159383552~29360128,209715200~4194304,218103808~4194304,230686720~4194304,239075328~8388608,251658240~4194304,260046848~16777216,289406976~4194304,297795584~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:35.153 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~12582912,33554432~8388608,54525952~12582912,71303168~20971520,125829120~12582912,159383552~29360128,209715200~4194304,218103808~4194304,230686720~4194304,239075328~8388608,251658240~4194304,260046848~16777216,289406976~4194304,297795584~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:35.160 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~20971520,33554432~12582912,54525952~37748736,125829120~12582912,159383552~33554432,209715200~16777216,230686720~4194304,239075328~8388608,251658240~4194304,260046848~16777216,289406976~4194304,297795584~8388608,327155712~12582912,348127232~58720256] 2024-03-07T03:48:35.160 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 15 diff [4194304~4194304,12582912~12582912,33554432~8388608,54525952~12582912,71303168~20971520,100663296~8388608,117440512~4194304,125829120~12582912,159383552~37748736,205520896~8388608,218103808~4194304,230686720~4194304,239075328~8388608,251658240~4194304,260046848~16777216,281018368~12582912,297795584~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:35.160 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~12582912,33554432~8388608,54525952~12582912,71303168~20971520,100663296~8388608,117440512~4194304,125829120~12582912,159383552~37748736,205520896~8388608,218103808~4194304,230686720~4194304,239075328~8388608,251658240~4194304,260046848~16777216,281018368~12582912,297795584~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~16777216,402653184~4194304] 2024-03-07T03:48:35.168 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~20971520,33554432~12582912,54525952~37748736,100663296~8388608,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~4194304,260046848~16777216,281018368~12582912,297795584~8388608,327155712~12582912,348127232~58720256] 2024-03-07T03:48:35.168 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 16 diff [4194304~4194304,12582912~12582912,33554432~8388608,54525952~12582912,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~8388608,218103808~4194304,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,327155712~8388608,348127232~29360128,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.168 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~12582912,33554432~8388608,54525952~12582912,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~8388608,218103808~4194304,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,327155712~8388608,348127232~29360128,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.177 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~20971520,33554432~12582912,54525952~41943040,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~8388608,327155712~12582912,348127232~62914560] 2024-03-07T03:48:35.178 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 17 diff [4194304~25165824,33554432~8388608,54525952~12582912,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,327155712~8388608,348127232~29360128,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.178 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,54525952~12582912,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,327155712~8388608,348127232~29360128,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.187 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,54525952~41943040,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,327155712~12582912,348127232~62914560] 2024-03-07T03:48:35.187 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 18 diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,322961408~12582912,343932928~54525952,402653184~8388608] 2024-03-07T03:48:35.188 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,322961408~12582912,343932928~54525952,402653184~8388608] 2024-03-07T03:48:35.199 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~46137344,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,322961408~16777216,343932928~67108864] 2024-03-07T03:48:35.199 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to 19 diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.199 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.211 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~46137344,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:35.211 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 12 diff [4194304~4194304,33554432~8388608,75497472~8388608,167772160~8388608,222298112~4194304,239075328~8388608,260046848~12582912,289406976~4194304,360710144~8388608] 2024-03-07T03:48:35.211 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,33554432~8388608,75497472~8388608,167772160~8388608,222298112~4194304,239075328~8388608,260046848~12582912,289406976~4194304,360710144~8388608] 2024-03-07T03:48:35.214 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~8388608,33554432~12582912,75497472~8388608,167772160~8388608,222298112~4194304,239075328~8388608,260046848~16777216,289406976~4194304,360710144~12582912] 2024-03-07T03:48:35.215 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 13 diff [4194304~4194304,33554432~8388608,62914560~4194304,75497472~8388608,159383552~29360128,222298112~4194304,230686720~4194304,239075328~8388608,260046848~12582912,289406976~4194304,297795584~4194304,360710144~8388608,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.215 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,33554432~8388608,62914560~4194304,75497472~8388608,159383552~29360128,222298112~4194304,230686720~4194304,239075328~8388608,260046848~12582912,289406976~4194304,297795584~4194304,360710144~8388608,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.219 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~8388608,33554432~12582912,62914560~8388608,75497472~8388608,159383552~33554432,222298112~4194304,230686720~4194304,239075328~8388608,260046848~16777216,289406976~4194304,297795584~8388608,360710144~12582912,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.219 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 14 diff [4194304~4194304,33554432~8388608,62914560~4194304,75497472~8388608,125829120~8388608,159383552~29360128,209715200~4194304,218103808~4194304,230686720~4194304,239075328~8388608,251658240~4194304,260046848~12582912,289406976~4194304,297795584~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.219 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,33554432~8388608,62914560~4194304,75497472~8388608,125829120~8388608,159383552~29360128,209715200~4194304,218103808~4194304,230686720~4194304,239075328~8388608,251658240~4194304,260046848~12582912,289406976~4194304,297795584~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.225 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~8388608,33554432~12582912,62914560~8388608,75497472~8388608,125829120~8388608,159383552~33554432,209715200~16777216,230686720~4194304,239075328~8388608,251658240~4194304,260046848~16777216,289406976~4194304,297795584~8388608,327155712~12582912,348127232~41943040,402653184~4194304] 2024-03-07T03:48:35.225 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 15 diff [4194304~4194304,16777216~4194304,33554432~8388608,58720256~8388608,75497472~8388608,100663296~8388608,117440512~4194304,125829120~12582912,159383552~37748736,205520896~8388608,218103808~4194304,230686720~4194304,239075328~8388608,251658240~4194304,260046848~12582912,281018368~12582912,297795584~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.225 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,16777216~4194304,33554432~8388608,58720256~8388608,75497472~8388608,100663296~8388608,117440512~4194304,125829120~12582912,159383552~37748736,205520896~8388608,218103808~4194304,230686720~4194304,239075328~8388608,251658240~4194304,260046848~12582912,281018368~12582912,297795584~4194304,327155712~8388608,348127232~20971520,373293056~4194304,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.232 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~8388608,16777216~4194304,33554432~12582912,58720256~12582912,75497472~8388608,100663296~8388608,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~4194304,260046848~16777216,281018368~12582912,297795584~8388608,327155712~12582912,348127232~41943040,402653184~4194304] 2024-03-07T03:48:35.232 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 16 diff [4194304~4194304,16777216~4194304,33554432~8388608,58720256~12582912,75497472~8388608,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~8388608,218103808~4194304,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,327155712~8388608,348127232~29360128,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.233 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,16777216~4194304,33554432~8388608,58720256~12582912,75497472~8388608,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~8388608,218103808~4194304,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,327155712~8388608,348127232~29360128,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.240 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~8388608,16777216~4194304,33554432~12582912,58720256~12582912,75497472~8388608,92274688~4194304,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~8388608,327155712~12582912,348127232~41943040,402653184~8388608] 2024-03-07T03:48:35.240 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 17 diff [4194304~8388608,16777216~12582912,33554432~8388608,58720256~12582912,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,327155712~8388608,348127232~29360128,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.240 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,16777216~12582912,33554432~8388608,58720256~12582912,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,327155712~8388608,348127232~29360128,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.248 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,58720256~12582912,75497472~20971520,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,327155712~12582912,348127232~41943040,402653184~8388608] 2024-03-07T03:48:35.249 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 18 diff [4194304~8388608,16777216~12582912,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,322961408~12582912,343932928~46137344,402653184~8388608] 2024-03-07T03:48:35.249 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,16777216~12582912,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,322961408~12582912,343932928~46137344,402653184~8388608] 2024-03-07T03:48:35.259 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,75497472~20971520,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,322961408~16777216,343932928~46137344,402653184~8388608] 2024-03-07T03:48:35.259 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to 19 diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.259 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.269 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,75497472~20971520,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:35.269 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 13 diff [62914560~4194304,159383552~8388608,176160768~12582912,230686720~4194304,297795584~4194304,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.269 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [62914560~4194304,159383552~8388608,176160768~12582912,230686720~4194304,297795584~4194304,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.273 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [62914560~8388608,159383552~8388608,176160768~16777216,230686720~4194304,297795584~8388608,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.273 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 14 diff [33554432~4194304,62914560~4194304,125829120~8388608,159383552~8388608,176160768~12582912,209715200~4194304,218103808~4194304,230686720~4194304,239075328~4194304,251658240~4194304,297795584~4194304,327155712~8388608,348127232~12582912,373293056~4194304,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.274 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [33554432~4194304,62914560~4194304,125829120~8388608,159383552~8388608,176160768~12582912,209715200~4194304,218103808~4194304,230686720~4194304,239075328~4194304,251658240~4194304,297795584~4194304,327155712~8388608,348127232~12582912,373293056~4194304,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.278 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [33554432~8388608,62914560~8388608,125829120~8388608,159383552~8388608,176160768~16777216,209715200~16777216,230686720~4194304,239075328~8388608,251658240~4194304,297795584~8388608,327155712~12582912,348127232~16777216,373293056~16777216,402653184~4194304] 2024-03-07T03:48:35.278 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 15 diff [16777216~4194304,33554432~4194304,58720256~8388608,100663296~8388608,117440512~4194304,125829120~12582912,159383552~37748736,205520896~8388608,218103808~4194304,230686720~4194304,239075328~4194304,251658240~4194304,281018368~8388608,297795584~4194304,327155712~8388608,348127232~12582912,373293056~4194304,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.278 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [16777216~4194304,33554432~4194304,58720256~8388608,100663296~8388608,117440512~4194304,125829120~12582912,159383552~37748736,205520896~8388608,218103808~4194304,230686720~4194304,239075328~4194304,251658240~4194304,281018368~8388608,297795584~4194304,327155712~8388608,348127232~12582912,373293056~4194304,381681664~8388608,402653184~4194304] 2024-03-07T03:48:35.284 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [16777216~4194304,33554432~8388608,58720256~12582912,100663296~8388608,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~4194304,281018368~8388608,297795584~8388608,327155712~12582912,348127232~16777216,373293056~16777216,402653184~4194304] 2024-03-07T03:48:35.284 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 16 diff [16777216~4194304,33554432~4194304,58720256~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~8388608,218103808~4194304,230686720~4194304,239075328~4194304,251658240~12582912,281018368~8388608,297795584~4194304,327155712~8388608,348127232~16777216,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.284 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [16777216~4194304,33554432~4194304,58720256~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~8388608,218103808~4194304,230686720~4194304,239075328~4194304,251658240~12582912,281018368~8388608,297795584~4194304,327155712~8388608,348127232~16777216,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.291 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [16777216~4194304,33554432~8388608,58720256~12582912,92274688~4194304,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~8388608,327155712~12582912,348127232~16777216,369098752~20971520,402653184~8388608] 2024-03-07T03:48:35.291 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 17 diff [4194304~8388608,16777216~12582912,33554432~4194304,58720256~12582912,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,327155712~8388608,348127232~16777216,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.291 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,16777216~12582912,33554432~4194304,58720256~12582912,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,327155712~8388608,348127232~16777216,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.299 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~8388608,58720256~12582912,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~25165824,281018368~8388608,297795584~8388608,327155712~12582912,348127232~16777216,369098752~20971520,402653184~8388608] 2024-03-07T03:48:35.299 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 18 diff [4194304~8388608,16777216~12582912,33554432~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,322961408~12582912,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.299 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,16777216~12582912,33554432~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,322961408~12582912,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.309 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~29360128,281018368~8388608,297795584~8388608,322961408~16777216,343932928~46137344,402653184~8388608] 2024-03-07T03:48:35.309 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to 19 diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.309 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.319 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~29360128,281018368~8388608,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:35.319 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to 14 diff [33554432~4194304,125829120~8388608,209715200~4194304,218103808~4194304,239075328~4194304,251658240~4194304,327155712~8388608,348127232~12582912,373293056~4194304] 2024-03-07T03:48:35.319 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [33554432~4194304,125829120~8388608,209715200~4194304,218103808~4194304,239075328~4194304,251658240~4194304,327155712~8388608,348127232~12582912,373293056~4194304] 2024-03-07T03:48:35.323 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [33554432~8388608,125829120~8388608,209715200~16777216,239075328~8388608,251658240~4194304,327155712~12582912,348127232~16777216,373293056~8388608] 2024-03-07T03:48:35.323 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to 15 diff [16777216~4194304,33554432~4194304,58720256~4194304,100663296~8388608,117440512~4194304,125829120~12582912,167772160~8388608,188743680~8388608,205520896~8388608,218103808~4194304,239075328~4194304,251658240~4194304,281018368~8388608,327155712~8388608,348127232~12582912,373293056~4194304] 2024-03-07T03:48:35.323 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [16777216~4194304,33554432~4194304,58720256~4194304,100663296~8388608,117440512~4194304,125829120~12582912,167772160~8388608,188743680~8388608,205520896~8388608,218103808~4194304,239075328~4194304,251658240~4194304,281018368~8388608,327155712~8388608,348127232~12582912,373293056~4194304] 2024-03-07T03:48:35.329 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [16777216~4194304,33554432~8388608,58720256~8388608,100663296~8388608,117440512~4194304,125829120~12582912,167772160~12582912,188743680~8388608,205520896~20971520,239075328~8388608,251658240~4194304,281018368~8388608,327155712~12582912,348127232~16777216,373293056~8388608] 2024-03-07T03:48:35.329 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to 16 diff [16777216~4194304,33554432~4194304,58720256~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,167772160~8388608,188743680~12582912,205520896~8388608,218103808~4194304,239075328~4194304,251658240~12582912,281018368~8388608,327155712~8388608,348127232~16777216,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.329 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [16777216~4194304,33554432~4194304,58720256~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,167772160~8388608,188743680~12582912,205520896~8388608,218103808~4194304,239075328~4194304,251658240~12582912,281018368~8388608,327155712~8388608,348127232~16777216,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.335 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [16777216~4194304,33554432~8388608,58720256~12582912,92274688~4194304,100663296~20971520,125829120~12582912,167772160~12582912,188743680~12582912,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,327155712~12582912,348127232~16777216,369098752~12582912,402653184~8388608] 2024-03-07T03:48:35.335 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to 17 diff [4194304~8388608,16777216~12582912,33554432~4194304,58720256~12582912,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,167772160~8388608,188743680~12582912,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,327155712~8388608,348127232~16777216,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.335 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,16777216~12582912,33554432~4194304,58720256~12582912,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,167772160~8388608,188743680~12582912,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,327155712~8388608,348127232~16777216,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.342 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~8388608,58720256~12582912,79691776~16777216,100663296~20971520,125829120~12582912,167772160~12582912,188743680~12582912,205520896~20971520,239075328~25165824,281018368~8388608,327155712~12582912,348127232~16777216,369098752~12582912,402653184~8388608] 2024-03-07T03:48:35.342 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to 18 diff [4194304~8388608,16777216~12582912,33554432~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,167772160~12582912,188743680~12582912,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,322961408~12582912,343932928~33554432,402653184~8388608] 2024-03-07T03:48:35.342 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,16777216~12582912,33554432~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,167772160~12582912,188743680~12582912,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,322961408~12582912,343932928~33554432,402653184~8388608] 2024-03-07T03:48:35.350 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,167772160~16777216,188743680~12582912,205520896~20971520,239075328~29360128,281018368~8388608,322961408~16777216,343932928~37748736,402653184~8388608] 2024-03-07T03:48:35.350 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to 19 diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~4194304,167772160~12582912,184549376~16777216,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,318767104~16777216,343932928~20971520,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.350 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~4194304,167772160~12582912,184549376~16777216,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,318767104~16777216,343932928~20971520,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.359 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,239075328~29360128,281018368~8388608,318767104~20971520,343932928~37748736,402653184~8388608] 2024-03-07T03:48:35.359 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to 15 diff [16777216~4194304,58720256~4194304,100663296~8388608,117440512~4194304,130023424~8388608,167772160~8388608,188743680~8388608,205520896~4194304,281018368~8388608] 2024-03-07T03:48:35.360 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [16777216~4194304,58720256~4194304,100663296~8388608,117440512~4194304,130023424~8388608,167772160~8388608,188743680~8388608,205520896~4194304,281018368~8388608] 2024-03-07T03:48:35.363 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [16777216~4194304,58720256~8388608,100663296~8388608,117440512~4194304,130023424~8388608,167772160~12582912,188743680~8388608,205520896~4194304,281018368~8388608] 2024-03-07T03:48:35.363 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to 16 diff [16777216~4194304,58720256~12582912,92274688~4194304,100663296~12582912,117440512~4194304,130023424~8388608,167772160~8388608,188743680~12582912,205520896~8388608,251658240~12582912,281018368~8388608,352321536~8388608,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.363 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [16777216~4194304,58720256~12582912,92274688~4194304,100663296~12582912,117440512~4194304,130023424~8388608,167772160~8388608,188743680~12582912,205520896~8388608,251658240~12582912,281018368~8388608,352321536~8388608,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.368 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [16777216~4194304,58720256~12582912,92274688~4194304,100663296~20971520,130023424~8388608,167772160~12582912,188743680~12582912,205520896~8388608,251658240~12582912,281018368~8388608,352321536~12582912,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.368 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to 17 diff [4194304~8388608,16777216~12582912,58720256~12582912,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,167772160~8388608,188743680~12582912,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,352321536~8388608,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.368 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,16777216~12582912,58720256~12582912,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,167772160~8388608,188743680~12582912,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,352321536~8388608,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.374 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,58720256~12582912,79691776~16777216,100663296~20971520,130023424~8388608,167772160~12582912,188743680~12582912,205520896~8388608,218103808~8388608,243269632~20971520,281018368~8388608,352321536~12582912,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.374 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to 18 diff [4194304~8388608,16777216~12582912,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,167772160~12582912,188743680~12582912,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,322961408~8388608,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.374 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,16777216~12582912,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,167772160~12582912,188743680~12582912,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,322961408~8388608,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.381 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,50331648~20971520,79691776~16777216,100663296~20971520,130023424~8388608,167772160~16777216,188743680~12582912,205520896~8388608,218103808~8388608,243269632~25165824,281018368~8388608,322961408~8388608,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.381 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to 19 diff [4194304~25165824,37748736~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,159383552~4194304,167772160~12582912,184549376~16777216,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.382 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,37748736~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,159383552~4194304,167772160~12582912,184549376~16777216,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.389 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~20971520,79691776~16777216,100663296~20971520,130023424~8388608,159383552~41943040,205520896~8388608,218103808~8388608,243269632~25165824,281018368~8388608,318767104~12582912,343932928~33554432,402653184~8388608] 2024-03-07T03:48:35.389 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to 16 diff [58720256~8388608,92274688~4194304,109051904~4194304,188743680~8388608,205520896~8388608,251658240~12582912,352321536~8388608,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.389 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [58720256~8388608,92274688~4194304,109051904~4194304,188743680~8388608,205520896~8388608,251658240~12582912,352321536~8388608,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.393 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [58720256~12582912,92274688~4194304,109051904~8388608,188743680~12582912,205520896~8388608,251658240~12582912,352321536~12582912,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.393 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to 17 diff [4194304~8388608,20971520~8388608,58720256~8388608,79691776~16777216,109051904~4194304,176160768~4194304,188743680~8388608,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,352321536~8388608,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.393 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,20971520~8388608,58720256~8388608,79691776~16777216,109051904~4194304,176160768~4194304,188743680~8388608,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,352321536~8388608,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.398 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~12582912,20971520~8388608,58720256~12582912,79691776~16777216,109051904~8388608,176160768~4194304,188743680~12582912,205520896~8388608,218103808~8388608,243269632~20971520,352321536~12582912,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.398 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to 18 diff [4194304~8388608,16777216~12582912,50331648~16777216,79691776~16777216,109051904~4194304,176160768~8388608,188743680~8388608,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,322961408~8388608,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.398 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,16777216~12582912,50331648~16777216,79691776~16777216,109051904~4194304,176160768~8388608,188743680~8388608,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,322961408~8388608,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.403 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,50331648~20971520,79691776~16777216,109051904~8388608,176160768~8388608,188743680~12582912,205520896~8388608,218103808~8388608,243269632~25165824,322961408~8388608,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.403 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to 19 diff [4194304~25165824,37748736~4194304,50331648~16777216,79691776~16777216,109051904~4194304,159383552~4194304,176160768~25165824,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.404 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,37748736~4194304,50331648~16777216,79691776~16777216,109051904~4194304,159383552~4194304,176160768~25165824,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.410 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~20971520,79691776~16777216,109051904~8388608,159383552~8388608,176160768~25165824,205520896~8388608,218103808~8388608,243269632~25165824,318767104~12582912,343932928~33554432,402653184~8388608] 2024-03-07T03:48:35.410 INFO:tasks.workunit.client.0.smithi145.stdout:from 16 to 17 diff [4194304~8388608,20971520~8388608,79691776~12582912,176160768~4194304,205520896~8388608,218103808~8388608,243269632~4194304,402653184~4194304] 2024-03-07T03:48:35.410 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,20971520~8388608,79691776~12582912,176160768~4194304,205520896~8388608,218103808~8388608,243269632~4194304,402653184~4194304] 2024-03-07T03:48:35.414 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~12582912,20971520~8388608,79691776~12582912,176160768~4194304,205520896~8388608,218103808~8388608,243269632~8388608,402653184~8388608] 2024-03-07T03:48:35.414 INFO:tasks.workunit.client.0.smithi145.stdout:from 16 to 18 diff [4194304~8388608,16777216~12582912,50331648~8388608,79691776~12582912,176160768~8388608,205520896~8388608,218103808~8388608,243269632~4194304,255852544~8388608,322961408~8388608,343932928~4194304,356515840~12582912,402653184~4194304] 2024-03-07T03:48:35.414 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~8388608,16777216~12582912,50331648~8388608,79691776~12582912,176160768~8388608,205520896~8388608,218103808~8388608,243269632~4194304,255852544~8388608,322961408~8388608,343932928~4194304,356515840~12582912,402653184~4194304] 2024-03-07T03:48:35.418 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,50331648~8388608,79691776~12582912,176160768~8388608,205520896~8388608,218103808~8388608,243269632~8388608,255852544~12582912,322961408~8388608,343932928~4194304,356515840~12582912,402653184~8388608] 2024-03-07T03:48:35.418 INFO:tasks.workunit.client.0.smithi145.stdout:from 16 to 19 diff [4194304~25165824,37748736~4194304,50331648~8388608,79691776~12582912,159383552~4194304,176160768~16777216,205520896~8388608,218103808~8388608,243269632~4194304,255852544~8388608,318767104~12582912,343932928~4194304,356515840~12582912,402653184~4194304] 2024-03-07T03:48:35.419 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,37748736~4194304,50331648~8388608,79691776~12582912,159383552~4194304,176160768~16777216,205520896~8388608,218103808~8388608,243269632~4194304,255852544~8388608,318767104~12582912,343932928~4194304,356515840~12582912,402653184~4194304] 2024-03-07T03:48:35.427 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~8388608,79691776~12582912,159383552~8388608,176160768~16777216,205520896~8388608,218103808~8388608,243269632~8388608,255852544~12582912,318767104~12582912,343932928~8388608,356515840~12582912,402653184~8388608] 2024-03-07T03:48:35.428 INFO:tasks.workunit.client.0.smithi145.stdout:from 17 to 18 diff [16777216~8388608,50331648~8388608,176160768~8388608,255852544~8388608,322961408~8388608,343932928~4194304,356515840~12582912] 2024-03-07T03:48:35.428 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [16777216~8388608,50331648~8388608,176160768~8388608,255852544~8388608,322961408~8388608,343932928~4194304,356515840~12582912] 2024-03-07T03:48:35.428 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [16777216~12582912,50331648~8388608,176160768~8388608,255852544~12582912,322961408~8388608,343932928~4194304,356515840~12582912] 2024-03-07T03:48:35.428 INFO:tasks.workunit.client.0.smithi145.stdout:from 17 to 19 diff [4194304~4194304,12582912~12582912,37748736~4194304,50331648~8388608,83886080~4194304,159383552~4194304,176160768~16777216,255852544~8388608,318767104~12582912,343932928~4194304,356515840~12582912] 2024-03-07T03:48:35.428 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~12582912,37748736~4194304,50331648~8388608,83886080~4194304,159383552~4194304,176160768~16777216,255852544~8388608,318767104~12582912,343932928~4194304,356515840~12582912] 2024-03-07T03:48:35.433 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~8388608,83886080~8388608,159383552~8388608,176160768~16777216,255852544~12582912,318767104~12582912,343932928~8388608,356515840~12582912] 2024-03-07T03:48:35.433 INFO:tasks.workunit.client.0.smithi145.stdout:from 18 to 19 diff [4194304~4194304,12582912~4194304,37748736~4194304,83886080~4194304,159383552~4194304,184549376~8388608,318767104~4194304,343932928~4194304,360710144~4194304] 2024-03-07T03:48:35.433 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~4194304,37748736~4194304,83886080~4194304,159383552~4194304,184549376~8388608,318767104~4194304,343932928~4194304,360710144~4194304] 2024-03-07T03:48:35.436 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~16777216,37748736~8388608,83886080~8388608,159383552~8388608,184549376~8388608,318767104~4194304,343932928~8388608,360710144~4194304] 2024-03-07T03:48:35.529 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.529 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.572 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:35.573 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.573 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.594 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:35.595 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.595 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.615 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:35.615 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.615 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.633 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:35.633 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.633 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.650 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:35.651 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.651 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.666 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:35.666 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.666 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.682 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:35.682 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.682 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.696 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:35.696 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.696 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.708 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:35.709 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.709 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.721 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~62914560,100663296~20971520,125829120~12582912,150994944~50331648,205520896~20971520,230686720~46137344,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:35.721 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.721 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:35.732 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~46137344,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:35.732 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.732 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.742 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,75497472~20971520,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:35.742 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.743 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:35.751 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~29360128,281018368~8388608,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:35.751 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~4194304,167772160~12582912,184549376~16777216,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,318767104~16777216,343932928~20971520,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.751 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~4194304,167772160~12582912,184549376~16777216,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,318767104~16777216,343932928~20971520,369098752~8388608,402653184~8388608] 2024-03-07T03:48:35.759 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,239075328~29360128,281018368~8388608,318767104~20971520,343932928~37748736,402653184~8388608] 2024-03-07T03:48:35.759 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to HEAD diff [4194304~25165824,37748736~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,159383552~4194304,167772160~12582912,184549376~16777216,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.759 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,37748736~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,159383552~4194304,167772160~12582912,184549376~16777216,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.765 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~20971520,79691776~16777216,100663296~20971520,130023424~8388608,159383552~41943040,205520896~8388608,218103808~8388608,243269632~25165824,281018368~8388608,318767104~12582912,343932928~33554432,402653184~8388608] 2024-03-07T03:48:35.766 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to HEAD diff [4194304~25165824,37748736~4194304,50331648~16777216,79691776~16777216,109051904~4194304,159383552~4194304,176160768~25165824,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.766 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,37748736~4194304,50331648~16777216,79691776~16777216,109051904~4194304,159383552~4194304,176160768~25165824,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:35.771 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~20971520,79691776~16777216,109051904~8388608,159383552~8388608,176160768~25165824,205520896~8388608,218103808~8388608,243269632~25165824,318767104~12582912,343932928~33554432,402653184~8388608] 2024-03-07T03:48:35.771 INFO:tasks.workunit.client.0.smithi145.stdout:from 16 to HEAD diff [4194304~25165824,37748736~4194304,50331648~8388608,79691776~12582912,159383552~4194304,176160768~16777216,205520896~8388608,218103808~8388608,243269632~4194304,255852544~8388608,318767104~12582912,343932928~4194304,356515840~12582912,402653184~4194304] 2024-03-07T03:48:35.771 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,37748736~4194304,50331648~8388608,79691776~12582912,159383552~4194304,176160768~16777216,205520896~8388608,218103808~8388608,243269632~4194304,255852544~8388608,318767104~12582912,343932928~4194304,356515840~12582912,402653184~4194304] 2024-03-07T03:48:35.775 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~8388608,79691776~12582912,159383552~8388608,176160768~16777216,205520896~8388608,218103808~8388608,243269632~8388608,255852544~12582912,318767104~12582912,343932928~8388608,356515840~12582912,402653184~8388608] 2024-03-07T03:48:35.775 INFO:tasks.workunit.client.0.smithi145.stdout:from 17 to HEAD diff [4194304~4194304,12582912~12582912,37748736~4194304,50331648~8388608,83886080~4194304,159383552~4194304,176160768~16777216,255852544~8388608,318767104~12582912,343932928~4194304,356515840~12582912] 2024-03-07T03:48:35.775 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~4194304,12582912~12582912,37748736~4194304,50331648~8388608,83886080~4194304,159383552~4194304,176160768~16777216,255852544~8388608,318767104~12582912,343932928~4194304,356515840~12582912] 2024-03-07T03:48:35.778 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~8388608,83886080~8388608,159383552~8388608,176160768~16777216,255852544~12582912,318767104~12582912,343932928~8388608,356515840~12582912] 2024-03-07T03:48:36.567 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.568 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.604 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:36.605 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.605 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.624 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:36.624 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.624 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.641 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:36.641 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.641 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.656 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:36.656 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.657 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.672 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:36.673 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.673 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.689 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:36.689 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.689 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.702 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:36.702 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.702 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.714 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:36.714 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.714 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.726 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:36.726 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.726 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.738 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~62914560,100663296~20971520,125829120~12582912,150994944~50331648,205520896~20971520,230686720~46137344,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:36.738 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.738 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:36.748 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~46137344,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:36.749 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:36.749 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:36.758 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,75497472~20971520,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:36.758 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:36.758 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:36.766 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~29360128,281018368~8388608,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:36.766 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~4194304,167772160~12582912,184549376~16777216,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,318767104~16777216,343932928~20971520,369098752~8388608,402653184~8388608] 2024-03-07T03:48:36.766 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~4194304,167772160~12582912,184549376~16777216,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,318767104~16777216,343932928~20971520,369098752~8388608,402653184~8388608] 2024-03-07T03:48:36.773 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,239075328~29360128,281018368~8388608,318767104~20971520,343932928~37748736,402653184~8388608] 2024-03-07T03:48:36.773 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to HEAD diff [4194304~25165824,37748736~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,159383552~4194304,167772160~12582912,184549376~16777216,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:36.773 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,37748736~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,159383552~4194304,167772160~12582912,184549376~16777216,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:36.779 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~20971520,79691776~16777216,100663296~20971520,130023424~8388608,159383552~41943040,205520896~8388608,218103808~8388608,243269632~25165824,281018368~8388608,318767104~12582912,343932928~33554432,402653184~8388608] 2024-03-07T03:48:36.780 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to HEAD diff [4194304~25165824,37748736~4194304,50331648~16777216,79691776~16777216,109051904~4194304,159383552~4194304,176160768~25165824,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:36.780 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,37748736~4194304,50331648~16777216,79691776~16777216,109051904~4194304,159383552~4194304,176160768~25165824,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:36.785 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~20971520,79691776~16777216,109051904~8388608,159383552~8388608,176160768~25165824,205520896~8388608,218103808~8388608,243269632~25165824,318767104~12582912,343932928~33554432,402653184~8388608] 2024-03-07T03:48:36.785 INFO:tasks.workunit.client.0.smithi145.stdout:from 16 to HEAD diff [4194304~25165824,37748736~4194304,50331648~8388608,79691776~12582912,159383552~4194304,176160768~16777216,205520896~8388608,218103808~8388608,243269632~4194304,255852544~8388608,318767104~12582912,343932928~4194304,356515840~12582912,402653184~4194304] 2024-03-07T03:48:36.785 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,37748736~4194304,50331648~8388608,79691776~12582912,159383552~4194304,176160768~16777216,205520896~8388608,218103808~8388608,243269632~4194304,255852544~8388608,318767104~12582912,343932928~4194304,356515840~12582912,402653184~4194304] 2024-03-07T03:48:36.788 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~8388608,79691776~12582912,159383552~8388608,176160768~16777216,205520896~8388608,218103808~8388608,243269632~8388608,255852544~12582912,318767104~12582912,343932928~8388608,356515840~12582912,402653184~8388608] 2024-03-07T03:48:36.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:36 smithi145 bash[79878]: cluster 2024-03-07T03:48:35.519574+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e1077: 8 total, 8 up, 8 in 2024-03-07T03:48:36.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:36 smithi145 bash[79878]: cluster 2024-03-07T03:48:35.651797+0000 mgr.y (mgr.34107) 1102 : cluster 0 pgmap v1804: 105 pgs: 105 active+clean; 824 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 175 KiB/s rd, 32 MiB/s wr, 282 op/s 2024-03-07T03:48:36.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:36 smithi145 bash[82329]: cluster 2024-03-07T03:48:35.519574+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e1077: 8 total, 8 up, 8 in 2024-03-07T03:48:36.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:36 smithi145 bash[82329]: cluster 2024-03-07T03:48:35.651797+0000 mgr.y (mgr.34107) 1102 : cluster 0 pgmap v1804: 105 pgs: 105 active+clean; 824 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 175 KiB/s rd, 32 MiB/s wr, 282 op/s 2024-03-07T03:48:36.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:36 smithi189 bash[66969]: cluster 2024-03-07T03:48:35.519574+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e1077: 8 total, 8 up, 8 in 2024-03-07T03:48:36.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:36 smithi189 bash[66969]: cluster 2024-03-07T03:48:35.651797+0000 mgr.y (mgr.34107) 1102 : cluster 0 pgmap v1804: 105 pgs: 105 active+clean; 824 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 175 KiB/s rd, 32 MiB/s wr, 282 op/s 2024-03-07T03:48:37.587 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.588 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.613 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:37.613 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.613 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.629 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:37.629 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.629 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.645 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:37.646 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.646 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.662 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:37.662 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.662 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.678 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:37.678 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.678 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.691 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:37.691 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.691 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.704 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:37.705 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.705 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.716 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:37.717 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.717 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.728 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:37.728 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.728 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.738 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~62914560,100663296~20971520,125829120~12582912,150994944~50331648,205520896~20971520,230686720~46137344,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:37.738 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.739 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:37.748 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~46137344,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:37.748 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:37.748 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:37.756 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,75497472~20971520,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:37.756 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:37.756 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:37.763 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~29360128,281018368~8388608,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:37.763 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~4194304,167772160~12582912,184549376~16777216,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,318767104~16777216,343932928~20971520,369098752~8388608,402653184~8388608] 2024-03-07T03:48:37.763 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~4194304,167772160~12582912,184549376~16777216,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,318767104~16777216,343932928~20971520,369098752~8388608,402653184~8388608] 2024-03-07T03:48:37.769 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,239075328~29360128,281018368~8388608,318767104~20971520,343932928~37748736,402653184~8388608] 2024-03-07T03:48:37.769 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to HEAD diff [4194304~25165824,37748736~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,159383552~4194304,167772160~12582912,184549376~16777216,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:37.769 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,37748736~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,159383552~4194304,167772160~12582912,184549376~16777216,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:37.773 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~20971520,79691776~16777216,100663296~20971520,130023424~8388608,159383552~41943040,205520896~8388608,218103808~8388608,243269632~25165824,281018368~8388608,318767104~12582912,343932928~33554432,402653184~8388608] 2024-03-07T03:48:37.773 INFO:tasks.workunit.client.0.smithi145.stdout:from 15 to HEAD diff [4194304~25165824,37748736~4194304,50331648~16777216,79691776~16777216,109051904~4194304,159383552~4194304,176160768~25165824,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:37.773 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,37748736~4194304,50331648~16777216,79691776~16777216,109051904~4194304,159383552~4194304,176160768~25165824,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:37.776 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~20971520,79691776~16777216,109051904~8388608,159383552~8388608,176160768~25165824,205520896~8388608,218103808~8388608,243269632~25165824,318767104~12582912,343932928~33554432,402653184~8388608] 2024-03-07T03:48:37.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:37 smithi145 bash[79878]: cluster 2024-03-07T03:48:36.537881+0000 mon.a (mon.0) 1931 : cluster 0 osdmap e1078: 8 total, 8 up, 8 in 2024-03-07T03:48:37.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:37 smithi145 bash[82329]: cluster 2024-03-07T03:48:36.537881+0000 mon.a (mon.0) 1931 : cluster 0 osdmap e1078: 8 total, 8 up, 8 in 2024-03-07T03:48:37.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:37 smithi189 bash[66969]: cluster 2024-03-07T03:48:36.537881+0000 mon.a (mon.0) 1931 : cluster 0 osdmap e1078: 8 total, 8 up, 8 in 2024-03-07T03:48:38.590 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.591 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.624 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:38.624 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.624 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.638 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:38.638 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.638 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.651 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:38.651 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.651 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.663 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:38.664 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.664 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.677 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:38.677 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.678 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.690 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:38.690 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.690 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.701 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:38.702 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.702 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.712 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:38.712 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.712 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.722 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:38.722 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.722 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.730 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~62914560,100663296~20971520,125829120~12582912,150994944~50331648,205520896~20971520,230686720~46137344,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:38.730 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.730 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:38.736 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~46137344,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:38.736 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:38.737 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:38.742 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,75497472~20971520,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:38.742 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:38.742 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:38.748 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~29360128,281018368~8388608,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:38.748 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~4194304,167772160~12582912,184549376~16777216,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,318767104~16777216,343932928~20971520,369098752~8388608,402653184~8388608] 2024-03-07T03:48:38.748 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~4194304,167772160~12582912,184549376~16777216,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,318767104~16777216,343932928~20971520,369098752~8388608,402653184~8388608] 2024-03-07T03:48:38.752 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,239075328~29360128,281018368~8388608,318767104~20971520,343932928~37748736,402653184~8388608] 2024-03-07T03:48:38.752 INFO:tasks.workunit.client.0.smithi145.stdout:from 14 to HEAD diff [4194304~25165824,37748736~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,159383552~4194304,167772160~12582912,184549376~16777216,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:38.752 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,37748736~4194304,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,130023424~8388608,159383552~4194304,167772160~12582912,184549376~16777216,205520896~8388608,218103808~8388608,243269632~4194304,251658240~12582912,281018368~8388608,318767104~12582912,343932928~4194304,352321536~25165824,402653184~8388608] 2024-03-07T03:48:38.755 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,37748736~8388608,50331648~20971520,79691776~16777216,100663296~20971520,130023424~8388608,159383552~41943040,205520896~8388608,218103808~8388608,243269632~25165824,281018368~8388608,318767104~12582912,343932928~33554432,402653184~8388608] 2024-03-07T03:48:38.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:38 smithi145 bash[79878]: cluster 2024-03-07T03:48:37.541507+0000 mon.a (mon.0) 1932 : cluster 0 osdmap e1079: 8 total, 8 up, 8 in 2024-03-07T03:48:38.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:38 smithi145 bash[79878]: cluster 2024-03-07T03:48:37.652488+0000 mgr.y (mgr.34107) 1103 : cluster 0 pgmap v1807: 105 pgs: 105 active+clean; 850 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 349 KiB/s rd, 31 MiB/s wr, 622 op/s 2024-03-07T03:48:38.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:38 smithi145 bash[82329]: cluster 2024-03-07T03:48:37.541507+0000 mon.a (mon.0) 1932 : cluster 0 osdmap e1079: 8 total, 8 up, 8 in 2024-03-07T03:48:38.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:38 smithi145 bash[82329]: cluster 2024-03-07T03:48:37.652488+0000 mgr.y (mgr.34107) 1103 : cluster 0 pgmap v1807: 105 pgs: 105 active+clean; 850 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 349 KiB/s rd, 31 MiB/s wr, 622 op/s 2024-03-07T03:48:38.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:38 smithi189 bash[66969]: cluster 2024-03-07T03:48:37.541507+0000 mon.a (mon.0) 1932 : cluster 0 osdmap e1079: 8 total, 8 up, 8 in 2024-03-07T03:48:38.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:38 smithi189 bash[66969]: cluster 2024-03-07T03:48:37.652488+0000 mgr.y (mgr.34107) 1103 : cluster 0 pgmap v1807: 105 pgs: 105 active+clean; 850 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 349 KiB/s rd, 31 MiB/s wr, 622 op/s 2024-03-07T03:48:39.585 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.585 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.622 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:39.623 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.623 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.633 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:39.633 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.633 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.643 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:39.643 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.643 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.652 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:39.653 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.653 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.662 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:39.662 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.662 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.670 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:39.670 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.670 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.678 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:39.678 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.678 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.685 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:39.685 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.685 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.692 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:39.692 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.692 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.698 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~62914560,100663296~20971520,125829120~12582912,150994944~50331648,205520896~20971520,230686720~46137344,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:39.698 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.698 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:39.703 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~46137344,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:39.703 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:39.703 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:39.707 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,75497472~20971520,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:39.707 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:39.707 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:39.712 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~29360128,281018368~8388608,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:39.712 INFO:tasks.workunit.client.0.smithi145.stdout:from 13 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~4194304,167772160~12582912,184549376~16777216,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,318767104~16777216,343932928~20971520,369098752~8388608,402653184~8388608] 2024-03-07T03:48:39.712 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~4194304,167772160~12582912,184549376~16777216,205520896~20971520,239075328~8388608,251658240~12582912,281018368~8388608,318767104~16777216,343932928~20971520,369098752~8388608,402653184~8388608] 2024-03-07T03:48:39.714 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,239075328~29360128,281018368~8388608,318767104~20971520,343932928~37748736,402653184~8388608] 2024-03-07T03:48:39.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:39 smithi145 bash[79878]: cluster 2024-03-07T03:48:38.551104+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e1080: 8 total, 8 up, 8 in 2024-03-07T03:48:39.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:39 smithi145 bash[82329]: cluster 2024-03-07T03:48:38.551104+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e1080: 8 total, 8 up, 8 in 2024-03-07T03:48:39.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:39 smithi189 bash[66969]: cluster 2024-03-07T03:48:38.551104+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e1080: 8 total, 8 up, 8 in 2024-03-07T03:48:40.596 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.596 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.640 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:40.640 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.640 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.649 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:40.649 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.649 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.657 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:40.657 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.657 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.665 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:40.666 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.666 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.673 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:40.673 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.674 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.680 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:40.680 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.680 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.687 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:40.687 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.687 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.692 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:40.692 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.693 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.698 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:40.699 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.699 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.703 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~62914560,100663296~20971520,125829120~12582912,150994944~50331648,205520896~20971520,230686720~46137344,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:40.703 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.703 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:40.707 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~46137344,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:40.707 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:40.707 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:40.711 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,75497472~20971520,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:40.711 INFO:tasks.workunit.client.0.smithi145.stdout:from 12 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:40.711 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,79691776~16777216,100663296~12582912,117440512~4194304,125829120~12582912,159383552~37748736,205520896~20971520,230686720~4194304,239075328~8388608,251658240~12582912,281018368~8388608,297795584~4194304,318767104~16777216,343932928~20971520,369098752~8388608,381681664~8388608,402653184~8388608] 2024-03-07T03:48:40.712 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,79691776~16777216,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~29360128,281018368~8388608,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:40.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:40 smithi145 bash[79878]: cluster 2024-03-07T03:48:39.561031+0000 mon.a (mon.0) 1934 : cluster 0 osdmap e1081: 8 total, 8 up, 8 in 2024-03-07T03:48:40.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:40 smithi145 bash[79878]: cluster 2024-03-07T03:48:39.652960+0000 mgr.y (mgr.34107) 1104 : cluster 0 pgmap v1810: 105 pgs: 1 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 101 active+clean; 849 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 596 KiB/s rd, 7.0 MiB/s wr, 1.10k op/s 2024-03-07T03:48:40.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:40 smithi145 bash[82329]: cluster 2024-03-07T03:48:39.561031+0000 mon.a (mon.0) 1934 : cluster 0 osdmap e1081: 8 total, 8 up, 8 in 2024-03-07T03:48:40.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:40 smithi145 bash[82329]: cluster 2024-03-07T03:48:39.652960+0000 mgr.y (mgr.34107) 1104 : cluster 0 pgmap v1810: 105 pgs: 1 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 101 active+clean; 849 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 596 KiB/s rd, 7.0 MiB/s wr, 1.10k op/s 2024-03-07T03:48:40.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:40 smithi189 bash[66969]: cluster 2024-03-07T03:48:39.561031+0000 mon.a (mon.0) 1934 : cluster 0 osdmap e1081: 8 total, 8 up, 8 in 2024-03-07T03:48:40.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:40 smithi189 bash[66969]: cluster 2024-03-07T03:48:39.652960+0000 mgr.y (mgr.34107) 1104 : cluster 0 pgmap v1810: 105 pgs: 1 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 101 active+clean; 849 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 596 KiB/s rd, 7.0 MiB/s wr, 1.10k op/s 2024-03-07T03:48:41.605 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.605 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.650 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:41.650 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.650 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.659 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:41.659 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.659 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.669 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:41.669 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.669 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.677 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:41.677 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.677 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.684 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:41.684 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.684 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.692 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:41.692 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.692 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.698 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:41.698 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.698 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.704 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:41.704 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.704 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.710 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:41.710 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.710 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.714 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~62914560,100663296~20971520,125829120~12582912,150994944~50331648,205520896~20971520,230686720~46137344,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:41.714 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.714 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:41.718 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~46137344,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:41.718 INFO:tasks.workunit.client.0.smithi145.stdout:from 11 to HEAD diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:41.718 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~20971520,75497472~12582912,92274688~4194304,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~20971520,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~8388608,402653184~8388608] 2024-03-07T03:48:41.720 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~20971520,75497472~20971520,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~46137344,402653184~8388608] 2024-03-07T03:48:41.817 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:41 smithi145 bash[79878]: cluster 2024-03-07T03:48:40.571920+0000 mon.a (mon.0) 1935 : cluster 0 osdmap e1082: 8 total, 8 up, 8 in 2024-03-07T03:48:41.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:41 smithi145 bash[82329]: cluster 2024-03-07T03:48:40.571920+0000 mon.a (mon.0) 1935 : cluster 0 osdmap e1082: 8 total, 8 up, 8 in 2024-03-07T03:48:41.837 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:41 smithi189 bash[66969]: cluster 2024-03-07T03:48:40.571920+0000 mon.a (mon.0) 1935 : cluster 0 osdmap e1082: 8 total, 8 up, 8 in 2024-03-07T03:48:42.582 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:48:42 smithi189 bash[59814]: ts=2024-03-07T03:48:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:48:42.624 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.624 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.668 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:42.668 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.668 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.675 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:42.676 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.676 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.683 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:42.683 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.683 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.690 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:42.690 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.690 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.696 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:42.696 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.696 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.702 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:42.702 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.702 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.708 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:42.708 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.708 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.711 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:42.711 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.711 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.715 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:42.715 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.715 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.718 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~62914560,100663296~20971520,125829120~12582912,150994944~50331648,205520896~20971520,230686720~46137344,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:42.718 INFO:tasks.workunit.client.0.smithi145.stdout:from 10 to HEAD diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.718 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~16777216,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~8388608,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:42.720 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~12582912,50331648~46137344,100663296~20971520,125829120~12582912,159383552~41943040,205520896~20971520,230686720~4194304,239075328~37748736,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:42.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:42 smithi189 bash[66969]: cluster 2024-03-07T03:48:41.581159+0000 mon.a (mon.0) 1936 : cluster 0 osdmap e1083: 8 total, 8 up, 8 in 2024-03-07T03:48:42.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:42 smithi189 bash[66969]: cluster 2024-03-07T03:48:41.653607+0000 mgr.y (mgr.34107) 1105 : cluster 0 pgmap v1813: 105 pgs: 2 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 98 active+clean; 821 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 434 KiB/s rd, 1.5 KiB/s wr, 869 op/s 2024-03-07T03:48:43.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:48:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:48:42] "GET /metrics HTTP/1.1" 200 36587 "" "Prometheus/2.43.0" 2024-03-07T03:48:43.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:42 smithi145 bash[79878]: cluster 2024-03-07T03:48:41.581159+0000 mon.a (mon.0) 1936 : cluster 0 osdmap e1083: 8 total, 8 up, 8 in 2024-03-07T03:48:43.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:42 smithi145 bash[79878]: cluster 2024-03-07T03:48:41.653607+0000 mgr.y (mgr.34107) 1105 : cluster 0 pgmap v1813: 105 pgs: 2 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 98 active+clean; 821 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 434 KiB/s rd, 1.5 KiB/s wr, 869 op/s 2024-03-07T03:48:43.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:42 smithi145 bash[82329]: cluster 2024-03-07T03:48:41.581159+0000 mon.a (mon.0) 1936 : cluster 0 osdmap e1083: 8 total, 8 up, 8 in 2024-03-07T03:48:43.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:42 smithi145 bash[82329]: cluster 2024-03-07T03:48:41.653607+0000 mgr.y (mgr.34107) 1105 : cluster 0 pgmap v1813: 105 pgs: 2 active+clean+snaptrim_wait, 5 active+clean+snaptrim, 98 active+clean; 821 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 434 KiB/s rd, 1.5 KiB/s wr, 869 op/s 2024-03-07T03:48:43.168 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.168 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.204 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:43.205 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.205 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.212 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:43.212 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.212 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.219 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:43.219 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.219 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.230 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:43.230 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.230 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.230 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:43.230 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.230 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.234 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:43.234 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.234 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.238 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:43.238 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.238 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.241 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:43.241 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.241 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.244 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:43.244 INFO:tasks.workunit.client.0.smithi145.stdout:from 9 to HEAD diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.244 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~20971520,71303168~25165824,100663296~12582912,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~25165824,281018368~12582912,297795584~4194304,318767104~16777216,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:43.246 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~25165824,33554432~62914560,100663296~20971520,125829120~12582912,150994944~50331648,205520896~20971520,230686720~46137344,281018368~12582912,297795584~8388608,318767104~20971520,343932928~67108864] 2024-03-07T03:48:43.840 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:43 smithi189 bash[66969]: cluster 2024-03-07T03:48:42.591721+0000 mon.a (mon.0) 1937 : cluster 0 osdmap e1084: 8 total, 8 up, 8 in 2024-03-07T03:48:43.840 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:43 smithi189 bash[66969]: cluster 2024-03-07T03:48:43.139130+0000 mon.a (mon.0) 1938 : cluster 0 osdmap e1085: 8 total, 8 up, 8 in 2024-03-07T03:48:44.067 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:43 smithi145 bash[79878]: cluster 2024-03-07T03:48:42.591721+0000 mon.a (mon.0) 1937 : cluster 0 osdmap e1084: 8 total, 8 up, 8 in 2024-03-07T03:48:44.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:43 smithi145 bash[79878]: cluster 2024-03-07T03:48:43.139130+0000 mon.a (mon.0) 1938 : cluster 0 osdmap e1085: 8 total, 8 up, 8 in 2024-03-07T03:48:44.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:43 smithi145 bash[82329]: cluster 2024-03-07T03:48:42.591721+0000 mon.a (mon.0) 1937 : cluster 0 osdmap e1084: 8 total, 8 up, 8 in 2024-03-07T03:48:44.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:43 smithi145 bash[82329]: cluster 2024-03-07T03:48:43.139130+0000 mon.a (mon.0) 1938 : cluster 0 osdmap e1085: 8 total, 8 up, 8 in 2024-03-07T03:48:44.173 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.174 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.211 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:44.211 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.211 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.217 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:44.218 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.218 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.224 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:44.224 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.224 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.229 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:44.230 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.230 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.235 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:44.235 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.235 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.238 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:44.238 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.239 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.242 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:44.242 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.242 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.244 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:44.245 INFO:tasks.workunit.client.0.smithi145.stdout:from 8 to HEAD diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.245 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [4194304~37748736,46137344~20971520,71303168~41943040,117440512~4194304,125829120~12582912,150994944~50331648,205520896~20971520,230686720~16777216,251658240~50331648,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:44.247 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [4194304~117440512,125829120~12582912,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:45.192 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.192 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.219 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:45.220 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.220 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.226 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:45.226 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.226 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.230 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:45.231 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.231 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.235 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:45.235 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.235 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.239 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:45.239 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.239 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.242 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:45.243 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.243 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.245 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:45.245 INFO:tasks.workunit.client.0.smithi145.stdout:from 7 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.245 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:45.247 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:45.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:45 smithi145 bash[79878]: cluster 2024-03-07T03:48:43.654578+0000 mgr.y (mgr.34107) 1106 : cluster 0 pgmap v1816: 105 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 102 active+clean; 773 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 342 KiB/s rd, 6.5 KiB/s wr, 603 op/s 2024-03-07T03:48:45.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:45 smithi145 bash[79878]: cluster 2024-03-07T03:48:44.141776+0000 mon.a (mon.0) 1939 : cluster 0 osdmap e1086: 8 total, 8 up, 8 in 2024-03-07T03:48:45.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:45 smithi145 bash[82329]: cluster 2024-03-07T03:48:43.654578+0000 mgr.y (mgr.34107) 1106 : cluster 0 pgmap v1816: 105 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 102 active+clean; 773 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 342 KiB/s rd, 6.5 KiB/s wr, 603 op/s 2024-03-07T03:48:45.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:45 smithi145 bash[82329]: cluster 2024-03-07T03:48:44.141776+0000 mon.a (mon.0) 1939 : cluster 0 osdmap e1086: 8 total, 8 up, 8 in 2024-03-07T03:48:45.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:45 smithi189 bash[66969]: cluster 2024-03-07T03:48:43.654578+0000 mgr.y (mgr.34107) 1106 : cluster 0 pgmap v1816: 105 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 102 active+clean; 773 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 342 KiB/s rd, 6.5 KiB/s wr, 603 op/s 2024-03-07T03:48:45.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:45 smithi189 bash[66969]: cluster 2024-03-07T03:48:44.141776+0000 mon.a (mon.0) 1939 : cluster 0 osdmap e1086: 8 total, 8 up, 8 in 2024-03-07T03:48:46.193 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.193 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.223 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:46.223 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.223 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.228 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:46.229 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.229 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.232 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:46.232 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.232 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.235 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:46.235 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.235 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.238 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:46.238 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.238 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.240 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:46.240 INFO:tasks.workunit.client.0.smithi145.stdout:from 6 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.241 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,150994944~50331648,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:46.242 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,150994944~50331648,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:46.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:46 smithi145 bash[79878]: cluster 2024-03-07T03:48:45.149478+0000 mon.a (mon.0) 1940 : cluster 0 osdmap e1087: 8 total, 8 up, 8 in 2024-03-07T03:48:46.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:46 smithi145 bash[82329]: cluster 2024-03-07T03:48:45.149478+0000 mon.a (mon.0) 1940 : cluster 0 osdmap e1087: 8 total, 8 up, 8 in 2024-03-07T03:48:46.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:46 smithi189 bash[66969]: cluster 2024-03-07T03:48:45.149478+0000 mon.a (mon.0) 1940 : cluster 0 osdmap e1087: 8 total, 8 up, 8 in 2024-03-07T03:48:47.189 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:47.189 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:47.227 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:47.227 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:47.227 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:47.236 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:47.236 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:47.236 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:47.239 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:47.239 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:47.239 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:47.242 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:47.242 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:47.242 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:47.244 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:47.245 INFO:tasks.workunit.client.0.smithi145.stdout:from 5 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:47.245 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~54525952,205520896~20971520,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:47.246 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~54525952,205520896~20971520,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:47.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:47 smithi145 bash[79878]: cluster 2024-03-07T03:48:45.655362+0000 mgr.y (mgr.34107) 1107 : cluster 0 pgmap v1819: 105 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 102 active+clean; 764 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 206 KiB/s rd, 5.0 KiB/s wr, 332 op/s 2024-03-07T03:48:47.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:47 smithi145 bash[79878]: cluster 2024-03-07T03:48:46.159361+0000 mon.a (mon.0) 1941 : cluster 0 osdmap e1088: 8 total, 8 up, 8 in 2024-03-07T03:48:47.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:47 smithi145 bash[82329]: cluster 2024-03-07T03:48:45.655362+0000 mgr.y (mgr.34107) 1107 : cluster 0 pgmap v1819: 105 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 102 active+clean; 764 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 206 KiB/s rd, 5.0 KiB/s wr, 332 op/s 2024-03-07T03:48:47.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:47 smithi145 bash[82329]: cluster 2024-03-07T03:48:46.159361+0000 mon.a (mon.0) 1941 : cluster 0 osdmap e1088: 8 total, 8 up, 8 in 2024-03-07T03:48:47.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:47 smithi189 bash[66969]: cluster 2024-03-07T03:48:45.655362+0000 mgr.y (mgr.34107) 1107 : cluster 0 pgmap v1819: 105 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 102 active+clean; 764 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 206 KiB/s rd, 5.0 KiB/s wr, 332 op/s 2024-03-07T03:48:47.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:47 smithi189 bash[66969]: cluster 2024-03-07T03:48:46.159361+0000 mon.a (mon.0) 1941 : cluster 0 osdmap e1088: 8 total, 8 up, 8 in 2024-03-07T03:48:48.162 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:48.162 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:48.205 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:48.206 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:48.206 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:48.221 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:48.221 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:48.221 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:48.226 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:48.226 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:48.226 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:48.229 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:48.229 INFO:tasks.workunit.client.0.smithi145.stdout:from 4 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:48.229 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~20971520,276824064~25165824,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:48.230 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:48.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:48 smithi145 bash[79878]: cluster 2024-03-07T03:48:47.159996+0000 mon.a (mon.0) 1942 : cluster 0 osdmap e1089: 8 total, 8 up, 8 in 2024-03-07T03:48:48.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:48 smithi145 bash[79878]: cluster 2024-03-07T03:48:47.656070+0000 mgr.y (mgr.34107) 1108 : cluster 0 pgmap v1822: 105 pgs: 105 active+clean; 719 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 6.2 KiB/s wr, 242 op/s 2024-03-07T03:48:48.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:48 smithi145 bash[79878]: audit 2024-03-07T03:48:47.682816+0000 mon.a (mon.0) 1943 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:48.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:48 smithi145 bash[79878]: cluster 2024-03-07T03:48:48.139941+0000 mon.a (mon.0) 1944 : cluster 0 osdmap e1090: 8 total, 8 up, 8 in 2024-03-07T03:48:48.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:48 smithi145 bash[82329]: cluster 2024-03-07T03:48:47.159996+0000 mon.a (mon.0) 1942 : cluster 0 osdmap e1089: 8 total, 8 up, 8 in 2024-03-07T03:48:48.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:48 smithi145 bash[82329]: cluster 2024-03-07T03:48:47.656070+0000 mgr.y (mgr.34107) 1108 : cluster 0 pgmap v1822: 105 pgs: 105 active+clean; 719 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 6.2 KiB/s wr, 242 op/s 2024-03-07T03:48:48.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:48 smithi145 bash[82329]: audit 2024-03-07T03:48:47.682816+0000 mon.a (mon.0) 1943 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:48.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:48 smithi145 bash[82329]: cluster 2024-03-07T03:48:48.139941+0000 mon.a (mon.0) 1944 : cluster 0 osdmap e1090: 8 total, 8 up, 8 in 2024-03-07T03:48:48.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:48 smithi189 bash[66969]: cluster 2024-03-07T03:48:47.159996+0000 mon.a (mon.0) 1942 : cluster 0 osdmap e1089: 8 total, 8 up, 8 in 2024-03-07T03:48:48.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:48 smithi189 bash[66969]: cluster 2024-03-07T03:48:47.656070+0000 mgr.y (mgr.34107) 1108 : cluster 0 pgmap v1822: 105 pgs: 105 active+clean; 719 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 151 KiB/s rd, 6.2 KiB/s wr, 242 op/s 2024-03-07T03:48:48.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:48 smithi189 bash[66969]: audit 2024-03-07T03:48:47.682816+0000 mon.a (mon.0) 1943 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:48.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:48 smithi189 bash[66969]: cluster 2024-03-07T03:48:48.139941+0000 mon.a (mon.0) 1944 : cluster 0 osdmap e1090: 8 total, 8 up, 8 in 2024-03-07T03:48:49.255 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:49.255 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:49.276 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:49.277 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:49.277 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:49.282 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:49.283 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:49.283 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:49.286 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:49.287 INFO:tasks.workunit.client.0.smithi145.stdout:from 3 to HEAD diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:49.287 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~4194304,125829120~16777216,146800640~79691776,230686720~16777216,251658240~54525952,314572800~20971520,343932928~33554432,381681664~16777216,402653184~8388608] 2024-03-07T03:48:49.288 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~121634816,125829120~16777216,146800640~79691776,230686720~75497472,314572800~25165824,343932928~67108864] 2024-03-07T03:48:50.265 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:50.265 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:50.290 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:50.291 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:50.291 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:50.294 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:50.294 INFO:tasks.workunit.client.0.smithi145.stdout:from 2 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:50.294 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~79691776,230686720~16777216,251658240~54525952,310378496~25165824,339738624~37748736,381681664~16777216,402653184~8388608] 2024-03-07T03:48:50.295 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~226492416,230686720~75497472,310378496~100663296] 2024-03-07T03:48:50.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:50 smithi145 bash[82329]: cluster 2024-03-07T03:48:49.223913+0000 mon.a (mon.0) 1945 : cluster 0 osdmap e1091: 8 total, 8 up, 8 in 2024-03-07T03:48:50.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:50 smithi145 bash[82329]: cluster 2024-03-07T03:48:49.656772+0000 mgr.y (mgr.34107) 1109 : cluster 0 pgmap v1825: 105 pgs: 105 active+clean; 657 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 160 KiB/s rd, 6.2 KiB/s wr, 262 op/s 2024-03-07T03:48:50.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:50 smithi145 bash[79878]: cluster 2024-03-07T03:48:49.223913+0000 mon.a (mon.0) 1945 : cluster 0 osdmap e1091: 8 total, 8 up, 8 in 2024-03-07T03:48:50.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:50 smithi145 bash[79878]: cluster 2024-03-07T03:48:49.656772+0000 mgr.y (mgr.34107) 1109 : cluster 0 pgmap v1825: 105 pgs: 105 active+clean; 657 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 160 KiB/s rd, 6.2 KiB/s wr, 262 op/s 2024-03-07T03:48:50.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:50 smithi189 bash[66969]: cluster 2024-03-07T03:48:49.223913+0000 mon.a (mon.0) 1945 : cluster 0 osdmap e1091: 8 total, 8 up, 8 in 2024-03-07T03:48:50.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:50 smithi189 bash[66969]: cluster 2024-03-07T03:48:49.656772+0000 mgr.y (mgr.34107) 1109 : cluster 0 pgmap v1825: 105 pgs: 105 active+clean; 657 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 160 KiB/s rd, 6.2 KiB/s wr, 262 op/s 2024-03-07T03:48:51.263 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:51.263 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:51.288 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:51.288 INFO:tasks.workunit.client.0.smithi145.stdout:from 1 to HEAD diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:51.288 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~20971520,117440512~25165824,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:51.291 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:51.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:51 smithi145 bash[79878]: cluster 2024-03-07T03:48:50.232642+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e1092: 8 total, 8 up, 8 in 2024-03-07T03:48:51.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:51 smithi145 bash[82329]: cluster 2024-03-07T03:48:50.232642+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e1092: 8 total, 8 up, 8 in 2024-03-07T03:48:51.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:51 smithi189 bash[66969]: cluster 2024-03-07T03:48:50.232642+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e1092: 8 total, 8 up, 8 in 2024-03-07T03:48:52.267 INFO:tasks.workunit.client.0.smithi145.stdout:from 0 to HEAD diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:52.267 INFO:tasks.workunit.client.0.smithi145.stdout: limited diff [0~88080384,92274688~50331648,146800640~100663296,251658240~54525952,310378496~67108864,381681664~16777216,402653184~8388608] 2024-03-07T03:48:52.294 INFO:tasks.workunit.client.0.smithi145.stdout: actual was [0~306184192,310378496~100663296] 2024-03-07T03:48:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:52 smithi145 bash[79878]: cluster 2024-03-07T03:48:51.233588+0000 mon.a (mon.0) 1947 : cluster 0 osdmap e1093: 8 total, 8 up, 8 in 2024-03-07T03:48:52.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:52 smithi145 bash[79878]: cluster 2024-03-07T03:48:51.657570+0000 mgr.y (mgr.34107) 1110 : cluster 0 pgmap v1828: 105 pgs: 105 active+clean; 556 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 45 KiB/s rd, 767 B/s wr, 89 op/s 2024-03-07T03:48:52.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:52 smithi145 bash[82329]: cluster 2024-03-07T03:48:51.233588+0000 mon.a (mon.0) 1947 : cluster 0 osdmap e1093: 8 total, 8 up, 8 in 2024-03-07T03:48:52.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:52 smithi145 bash[82329]: cluster 2024-03-07T03:48:51.657570+0000 mgr.y (mgr.34107) 1110 : cluster 0 pgmap v1828: 105 pgs: 105 active+clean; 556 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 45 KiB/s rd, 767 B/s wr, 89 op/s 2024-03-07T03:48:52.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:52 smithi189 bash[66969]: cluster 2024-03-07T03:48:51.233588+0000 mon.a (mon.0) 1947 : cluster 0 osdmap e1093: 8 total, 8 up, 8 in 2024-03-07T03:48:52.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:52 smithi189 bash[66969]: cluster 2024-03-07T03:48:51.657570+0000 mgr.y (mgr.34107) 1110 : cluster 0 pgmap v1828: 105 pgs: 105 active+clean; 556 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 45 KiB/s rd, 767 B/s wr, 89 op/s 2024-03-07T03:48:52.588 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:48:52 smithi189 bash[59814]: ts=2024-03-07T03:48:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:48:53.067 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:48:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:48:52] "GET /metrics HTTP/1.1" 200 36591 "" "Prometheus/2.43.0" 2024-03-07T03:48:53.205 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/1.DiffIterateStress (40007 ms) 2024-03-07T03:48:53.206 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParent 2024-03-07T03:48:53.206 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:48:53.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:53 smithi145 bash[79878]: cluster 2024-03-07T03:48:52.254287+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e1094: 8 total, 8 up, 8 in 2024-03-07T03:48:53.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:53 smithi145 bash[79878]: cluster 2024-03-07T03:48:53.144506+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e1095: 8 total, 8 up, 8 in 2024-03-07T03:48:53.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:53 smithi145 bash[82329]: cluster 2024-03-07T03:48:52.254287+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e1094: 8 total, 8 up, 8 in 2024-03-07T03:48:53.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:53 smithi145 bash[82329]: cluster 2024-03-07T03:48:53.144506+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e1095: 8 total, 8 up, 8 in 2024-03-07T03:48:53.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:53 smithi189 bash[66969]: cluster 2024-03-07T03:48:52.254287+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e1094: 8 total, 8 up, 8 in 2024-03-07T03:48:53.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:53 smithi189 bash[66969]: cluster 2024-03-07T03:48:53.144506+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e1095: 8 total, 8 up, 8 in 2024-03-07T03:48:54.524 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/1.DiffIterateParent (1318 ms) 2024-03-07T03:48:54.524 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/1.DiffIterateIgnoreParent 2024-03-07T03:48:54.524 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:48:54.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:54 smithi145 bash[79878]: cluster 2024-03-07T03:48:53.658586+0000 mgr.y (mgr.34107) 1111 : cluster 0 pgmap v1831: 105 pgs: 105 active+clean; 478 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 99 KiB/s rd, 2.0 MiB/s wr, 159 op/s 2024-03-07T03:48:54.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:54 smithi145 bash[82329]: cluster 2024-03-07T03:48:53.658586+0000 mgr.y (mgr.34107) 1111 : cluster 0 pgmap v1831: 105 pgs: 105 active+clean; 478 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 99 KiB/s rd, 2.0 MiB/s wr, 159 op/s 2024-03-07T03:48:54.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:54 smithi189 bash[66969]: cluster 2024-03-07T03:48:53.658586+0000 mgr.y (mgr.34107) 1111 : cluster 0 pgmap v1831: 105 pgs: 105 active+clean; 478 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 99 KiB/s rd, 2.0 MiB/s wr, 159 op/s 2024-03-07T03:48:55.545 INFO:tasks.workunit.client.0.smithi145.stdout: wrote [0~20971520] to clone 2024-03-07T03:48:55.546 INFO:tasks.workunit.client.0.smithi145.stdout: diff was [0~20971520] 2024-03-07T03:48:55.555 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/1.DiffIterateIgnoreParent (1031 ms) 2024-03-07T03:48:55.555 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/1.DiffIterateCallbackError 2024-03-07T03:48:55.556 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:48:55.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:55 smithi145 bash[79878]: cluster 2024-03-07T03:48:54.266329+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e1096: 8 total, 8 up, 8 in 2024-03-07T03:48:55.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:55 smithi145 bash[82329]: cluster 2024-03-07T03:48:54.266329+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e1096: 8 total, 8 up, 8 in 2024-03-07T03:48:55.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:55 smithi189 bash[66969]: cluster 2024-03-07T03:48:54.266329+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e1096: 8 total, 8 up, 8 in 2024-03-07T03:48:55.610 INFO:tasks.workunit.client.0.smithi145.stdout: wrote [1325129~19909,1976392~66052,3867473~76063,6188358~92595,14104603~23072,14184412~86260,16596304~100268,16953630~86856,17100761~22109,18999984~75848] 2024-03-07T03:48:55.639 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/1.DiffIterateCallbackError (84 ms) 2024-03-07T03:48:55.639 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParentDiscard 2024-03-07T03:48:55.639 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:48:56.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:56 smithi145 bash[79878]: cluster 2024-03-07T03:48:55.267578+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e1097: 8 total, 8 up, 8 in 2024-03-07T03:48:56.592 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:56 smithi145 bash[79878]: cluster 2024-03-07T03:48:55.659271+0000 mgr.y (mgr.34107) 1112 : cluster 0 pgmap v1834: 105 pgs: 105 active+clean; 471 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 98 KiB/s rd, 3.0 MiB/s wr, 140 op/s 2024-03-07T03:48:56.592 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:56 smithi145 bash[82329]: cluster 2024-03-07T03:48:55.267578+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e1097: 8 total, 8 up, 8 in 2024-03-07T03:48:56.592 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:56 smithi145 bash[82329]: cluster 2024-03-07T03:48:55.659271+0000 mgr.y (mgr.34107) 1112 : cluster 0 pgmap v1834: 105 pgs: 105 active+clean; 471 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 98 KiB/s rd, 3.0 MiB/s wr, 140 op/s 2024-03-07T03:48:56.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:56 smithi189 bash[66969]: cluster 2024-03-07T03:48:55.267578+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e1097: 8 total, 8 up, 8 in 2024-03-07T03:48:56.593 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:56 smithi189 bash[66969]: cluster 2024-03-07T03:48:55.659271+0000 mgr.y (mgr.34107) 1112 : cluster 0 pgmap v1834: 105 pgs: 105 active+clean; 471 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 98 KiB/s rd, 3.0 MiB/s wr, 140 op/s 2024-03-07T03:48:57.533 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/1.DiffIterateParentDiscard (1894 ms) 2024-03-07T03:48:57.533 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnalignedSmall 2024-03-07T03:48:57.533 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:48:57.567 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:57 smithi145 bash[79878]: cluster 2024-03-07T03:48:56.288529+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e1098: 8 total, 8 up, 8 in 2024-03-07T03:48:57.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:57 smithi145 bash[82329]: cluster 2024-03-07T03:48:56.288529+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e1098: 8 total, 8 up, 8 in 2024-03-07T03:48:57.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:57 smithi189 bash[66969]: cluster 2024-03-07T03:48:56.288529+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e1098: 8 total, 8 up, 8 in 2024-03-07T03:48:57.646 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnalignedSmall (113 ms) 2024-03-07T03:48:57.646 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnaligned 2024-03-07T03:48:57.647 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:48:57.768 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnaligned (122 ms) 2024-03-07T03:48:57.769 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/1.DiffIterateTryAcquireLock 2024-03-07T03:48:57.769 INFO:tasks.workunit.client.0.smithi145.stdout:using new format! 2024-03-07T03:48:58.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:58 smithi145 bash[79878]: cluster 2024-03-07T03:48:57.296364+0000 mon.a (mon.0) 1953 : cluster 0 osdmap e1099: 8 total, 8 up, 8 in 2024-03-07T03:48:58.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:58 smithi145 bash[79878]: cluster 2024-03-07T03:48:57.659968+0000 mgr.y (mgr.34107) 1113 : cluster 0 pgmap v1837: 105 pgs: 105 active+clean; 475 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 7.1 MiB/s wr, 182 op/s 2024-03-07T03:48:58.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:48:58 smithi145 bash[79878]: cluster 2024-03-07T03:48:58.145435+0000 mon.a (mon.0) 1954 : cluster 0 osdmap e1100: 8 total, 8 up, 8 in 2024-03-07T03:48:58.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:58 smithi145 bash[82329]: cluster 2024-03-07T03:48:57.296364+0000 mon.a (mon.0) 1953 : cluster 0 osdmap e1099: 8 total, 8 up, 8 in 2024-03-07T03:48:58.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:58 smithi145 bash[82329]: cluster 2024-03-07T03:48:57.659968+0000 mgr.y (mgr.34107) 1113 : cluster 0 pgmap v1837: 105 pgs: 105 active+clean; 475 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 7.1 MiB/s wr, 182 op/s 2024-03-07T03:48:58.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:48:58 smithi145 bash[82329]: cluster 2024-03-07T03:48:58.145435+0000 mon.a (mon.0) 1954 : cluster 0 osdmap e1100: 8 total, 8 up, 8 in 2024-03-07T03:48:58.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:58 smithi189 bash[66969]: cluster 2024-03-07T03:48:57.296364+0000 mon.a (mon.0) 1953 : cluster 0 osdmap e1099: 8 total, 8 up, 8 in 2024-03-07T03:48:58.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:58 smithi189 bash[66969]: cluster 2024-03-07T03:48:57.659968+0000 mgr.y (mgr.34107) 1113 : cluster 0 pgmap v1837: 105 pgs: 105 active+clean; 475 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 7.1 MiB/s wr, 182 op/s 2024-03-07T03:48:58.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:48:58 smithi189 bash[66969]: cluster 2024-03-07T03:48:58.145435+0000 mon.a (mon.0) 1954 : cluster 0 osdmap e1100: 8 total, 8 up, 8 in 2024-03-07T03:49:01.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:00 smithi145 bash[79878]: cluster 2024-03-07T03:48:59.660972+0000 mgr.y (mgr.34107) 1114 : cluster 0 pgmap v1839: 105 pgs: 105 active+clean; 505 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 8.9 MiB/s wr, 214 op/s 2024-03-07T03:49:01.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:00 smithi145 bash[82329]: cluster 2024-03-07T03:48:59.660972+0000 mgr.y (mgr.34107) 1114 : cluster 0 pgmap v1839: 105 pgs: 105 active+clean; 505 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 8.9 MiB/s wr, 214 op/s 2024-03-07T03:49:01.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:00 smithi189 bash[66969]: cluster 2024-03-07T03:48:59.660972+0000 mgr.y (mgr.34107) 1114 : cluster 0 pgmap v1839: 105 pgs: 105 active+clean; 505 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 8.9 MiB/s wr, 214 op/s 2024-03-07T03:49:02.588 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:49:02 smithi189 bash[59814]: ts=2024-03-07T03:49:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:49:02.883 INFO:tasks.workunit.client.0.smithi145.stdout:[ OK ] DiffIterateTest/1.DiffIterateTryAcquireLock (5115 ms) 2024-03-07T03:49:02.883 INFO:tasks.workunit.client.0.smithi145.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStriping 2024-03-07T03:49:02.883 INFO:tasks.workunit.client.0.smithi145.stdout:./src/test/librbd/test_librbd.cc:8289: Skipped 2024-03-07T03:49:02.883 INFO:tasks.workunit.client.0.smithi145.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:49:02.883 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] DiffIterateTest/1.DiffIterateStriping (0 ms) 2024-03-07T03:49:03.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:49:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:49:02] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.43.0" 2024-03-07T03:49:03.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:02 smithi145 bash[79878]: cluster 2024-03-07T03:49:01.662030+0000 mgr.y (mgr.34107) 1115 : cluster 0 pgmap v1840: 105 pgs: 105 active+clean; 547 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 10 MiB/s wr, 257 op/s 2024-03-07T03:49:03.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:02 smithi145 bash[79878]: audit 2024-03-07T03:49:02.683290+0000 mon.a (mon.0) 1955 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:03.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:02 smithi145 bash[82329]: cluster 2024-03-07T03:49:01.662030+0000 mgr.y (mgr.34107) 1115 : cluster 0 pgmap v1840: 105 pgs: 105 active+clean; 547 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 10 MiB/s wr, 257 op/s 2024-03-07T03:49:03.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:02 smithi145 bash[82329]: audit 2024-03-07T03:49:02.683290+0000 mon.a (mon.0) 1955 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:03.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:02 smithi189 bash[66969]: cluster 2024-03-07T03:49:01.662030+0000 mgr.y (mgr.34107) 1115 : cluster 0 pgmap v1840: 105 pgs: 105 active+clean; 547 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 10 MiB/s wr, 257 op/s 2024-03-07T03:49:03.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:02 smithi189 bash[66969]: audit 2024-03-07T03:49:02.683290+0000 mon.a (mon.0) 1955 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:03.152 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] 13 tests from DiffIterateTest/1 (63705 ms total) 2024-03-07T03:49:03.152 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:49:03.152 INFO:tasks.workunit.client.0.smithi145.stdout:[----------] Global test environment tear-down 2024-03-07T03:49:03.152 INFO:tasks.workunit.client.0.smithi145.stdout:[==========] 476 tests from 24 test suites ran. (2275800 ms total) 2024-03-07T03:49:03.152 INFO:tasks.workunit.client.0.smithi145.stdout:[ PASSED ] 430 tests. 2024-03-07T03:49:03.152 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] 46 tests, listed below: 2024-03-07T03:49:03.152 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalStress.DiscardWithPruneWriteOverlap 2024-03-07T03:49:03.152 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.AioDiscardEvent 2024-03-07T03:49:03.152 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.AioWriteEvent 2024-03-07T03:49:03.152 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.AioFlushEvent 2024-03-07T03:49:03.152 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.SnapCreate 2024-03-07T03:49:03.152 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.SnapProtect 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.SnapUnprotect 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.SnapRename 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.SnapRollback 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.SnapRemove 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.Rename 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.Resize 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.Flatten 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.UpdateFeatures 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.MetadataSet 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.MetadataRemove 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalReplay.ObjectPosition 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalEntries.AioWrite 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalEntries.AioDiscard 2024-03-07T03:49:03.153 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalEntries.AioDiscardWithPrune 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestJournalEntries.AioFlush 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestOperations.DisableJournalingCorrupt 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMigration.MirroringSamePool 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMigration.MirroringAbort 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMigration.MirroringOtherPoolDisabled 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMigration.MirroringOtherPoolEnabled 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMigration.MirroringPool 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.Snaps_StrippingLargerDstObjSize 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.Clone_StrippingLargerDstObjSize 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.Stress_StrippingLargerDstObjSize 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize 2024-03-07T03:49:03.154 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.Clone_StrippingSmallerDstObjSize 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestDeepCopy.Stress_StrippingSmallerDstObjSize 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMirroring.DisableImageMirrorWithPeer 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestMirroring.DisableJournalingWithPeer 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestInternal.DiscardCopyup 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestInternal.TestCoR 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestLibRBD.EncryptionLoadBadStripePattern 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestLibRBD.EnableJournalingViaLockOwner 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] TestLibRBD.ExclusiveLockReadTransition 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] DiffIterateTest/0.DiffIterateStriping 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout:[ SKIPPED ] DiffIterateTest/1.DiffIterateStriping 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:49:03.155 INFO:tasks.workunit.client.0.smithi145.stdout: YOU HAVE 2 DISABLED TESTS 2024-03-07T03:49:03.156 INFO:tasks.workunit.client.0.smithi145.stdout: 2024-03-07T03:49:03.216 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-03-07T03:49:03.216 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-03-07T03:49:03.275 INFO:tasks.workunit:Stopping ['rbd/test_librbd.sh'] on client.0... 2024-03-07T03:49:03.275 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-03-07T03:49:03.545 DEBUG:teuthology.parallel:result is None 2024-03-07T03:49:03.546 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-03-07T03:49:03.557 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-03-07T03:49:03.558 DEBUG:teuthology.orchestra.run.smithi145:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-03-07T03:49:03.615 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-03-07T03:49:03.616 INFO:teuthology.task.sequential:In sequential, running task print... 2024-03-07T03:49:03.616 INFO:teuthology.task.print:**** done rbd/test_librbd.sh 7-workload 2024-03-07T03:49:03.616 DEBUG:teuthology.parallel:result is None 2024-03-07T03:49:03.616 INFO:teuthology.run_tasks:Running task print... 2024-03-07T03:49:03.627 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-03-07T03:49:03.627 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-03-07T03:49:03.659 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-07T03:49:03.659 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-07T03:49:03.660 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph mgr dump --format=json 2024-03-07T03:49:04.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:04 smithi145 bash[79878]: cluster 2024-03-07T03:49:03.147410+0000 mon.a (mon.0) 1956 : cluster 0 osdmap e1101: 8 total, 8 up, 8 in 2024-03-07T03:49:04.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:04 smithi145 bash[82329]: cluster 2024-03-07T03:49:03.147410+0000 mon.a (mon.0) 1956 : cluster 0 osdmap e1101: 8 total, 8 up, 8 in 2024-03-07T03:49:04.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:04 smithi189 bash[66969]: cluster 2024-03-07T03:49:03.147410+0000 mon.a (mon.0) 1956 : cluster 0 osdmap e1101: 8 total, 8 up, 8 in 2024-03-07T03:49:05.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:05 smithi145 bash[79878]: cluster 2024-03-07T03:49:03.662585+0000 mgr.y (mgr.34107) 1116 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:49:05.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:05 smithi145 bash[82329]: cluster 2024-03-07T03:49:03.662585+0000 mgr.y (mgr.34107) 1116 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:49:05.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:05 smithi189 bash[66969]: cluster 2024-03-07T03:49:03.662585+0000 mgr.y (mgr.34107) 1116 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:49:06.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:06 smithi145 bash[79878]: cluster 2024-03-07T03:49:05.663416+0000 mgr.y (mgr.34107) 1117 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:49:06.568 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:06 smithi145 bash[82329]: cluster 2024-03-07T03:49:05.663416+0000 mgr.y (mgr.34107) 1117 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:49:06.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:06 smithi189 bash[66969]: cluster 2024-03-07T03:49:05.663416+0000 mgr.y (mgr.34107) 1117 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:49:08.393 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:49:09.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:08 smithi145 bash[79878]: cluster 2024-03-07T03:49:07.664361+0000 mgr.y (mgr.34107) 1118 : cluster 0 pgmap v1844: 73 pgs: 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:49:09.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:08 smithi145 bash[82329]: cluster 2024-03-07T03:49:07.664361+0000 mgr.y (mgr.34107) 1118 : cluster 0 pgmap v1844: 73 pgs: 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:49:09.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:08 smithi189 bash[66969]: cluster 2024-03-07T03:49:07.664361+0000 mgr.y (mgr.34107) 1118 : cluster 0 pgmap v1844: 73 pgs: 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:49:11.037 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:10 smithi145 bash[82329]: cluster 2024-03-07T03:49:09.665261+0000 mgr.y (mgr.34107) 1119 : cluster 0 pgmap v1845: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:11.038 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:10 smithi145 bash[79878]: cluster 2024-03-07T03:49:09.665261+0000 mgr.y (mgr.34107) 1119 : cluster 0 pgmap v1845: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:11.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:10 smithi189 bash[66969]: cluster 2024-03-07T03:49:09.665261+0000 mgr.y (mgr.34107) 1119 : cluster 0 pgmap v1845: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:12.406 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:49:12.588 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:49:12 smithi189 bash[59814]: ts=2024-03-07T03:49:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:49:13.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:49:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:49:12] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.43.0" 2024-03-07T03:49:13.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:12 smithi145 bash[79878]: cluster 2024-03-07T03:49:11.666194+0000 mgr.y (mgr.34107) 1120 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:13.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:12 smithi145 bash[79878]: audit 2024-03-07T03:49:12.402881+0000 mon.a (mon.0) 1957 : audit 0 from='client.? 172.21.15.145:0/1566722139' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:49:13.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:12 smithi145 bash[82329]: cluster 2024-03-07T03:49:11.666194+0000 mgr.y (mgr.34107) 1120 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:13.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:12 smithi145 bash[82329]: audit 2024-03-07T03:49:12.402881+0000 mon.a (mon.0) 1957 : audit 0 from='client.? 172.21.15.145:0/1566722139' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:49:13.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:12 smithi189 bash[66969]: cluster 2024-03-07T03:49:11.666194+0000 mgr.y (mgr.34107) 1120 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:13.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:12 smithi189 bash[66969]: audit 2024-03-07T03:49:12.402881+0000 mon.a (mon.0) 1957 : audit 0 from='client.? 172.21.15.145:0/1566722139' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:49:13.225 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":38,"flags":0,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6800","nonce":2065626473},{"type":"v1","addr":"172.21.15.145:6801","nonce":2065626473}]},"active_addr":"172.21.15.145:6801/2065626473","active_change":"2024-03-07T03:15:02.496927+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":24748,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.145:8443/","prometheus":"http://172.21.15.145:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":282,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":385912723}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":3362698539}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":2492857125}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.145:0","nonce":2757936465}]}]} 2024-03-07T03:49:13.229 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-07T03:49:13.230 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-07T03:49:13.230 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:49:15.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:14 smithi145 bash[79878]: cluster 2024-03-07T03:49:13.667103+0000 mgr.y (mgr.34107) 1121 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:15.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:14 smithi145 bash[82329]: cluster 2024-03-07T03:49:13.667103+0000 mgr.y (mgr.34107) 1121 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:15.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:14 smithi189 bash[66969]: cluster 2024-03-07T03:49:13.667103+0000 mgr.y (mgr.34107) 1121 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:17.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:16 smithi145 bash[79878]: cluster 2024-03-07T03:49:15.667901+0000 mgr.y (mgr.34107) 1122 : cluster 0 pgmap v1848: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:17.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:16 smithi145 bash[82329]: cluster 2024-03-07T03:49:15.667901+0000 mgr.y (mgr.34107) 1122 : cluster 0 pgmap v1848: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:17.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:16 smithi189 bash[66969]: cluster 2024-03-07T03:49:15.667901+0000 mgr.y (mgr.34107) 1122 : cluster 0 pgmap v1848: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:17.966 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:49:18.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:17 smithi145 bash[79878]: audit 2024-03-07T03:49:17.683768+0000 mon.a (mon.0) 1958 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:18.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:17 smithi145 bash[82329]: audit 2024-03-07T03:49:17.683768+0000 mon.a (mon.0) 1958 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:18.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:17 smithi189 bash[66969]: audit 2024-03-07T03:49:17.683768+0000 mon.a (mon.0) 1958 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:19.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:18 smithi145 bash[79878]: cluster 2024-03-07T03:49:17.668834+0000 mgr.y (mgr.34107) 1123 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:19.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:18 smithi145 bash[82329]: cluster 2024-03-07T03:49:17.668834+0000 mgr.y (mgr.34107) 1123 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:19.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:18 smithi189 bash[66969]: cluster 2024-03-07T03:49:17.668834+0000 mgr.y (mgr.34107) 1123 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:21.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:20 smithi145 bash[79878]: cluster 2024-03-07T03:49:19.669695+0000 mgr.y (mgr.34107) 1124 : cluster 0 pgmap v1850: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:21.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:20 smithi145 bash[82329]: cluster 2024-03-07T03:49:19.669695+0000 mgr.y (mgr.34107) 1124 : cluster 0 pgmap v1850: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:21.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:20 smithi189 bash[66969]: cluster 2024-03-07T03:49:19.669695+0000 mgr.y (mgr.34107) 1124 : cluster 0 pgmap v1850: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:21.296 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:49:21.297 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":1101,"fsid":"8947a0f8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:55:42.039360+0000","modified":"2024-03-07T03:49:03.137663+0000","last_up_change":"2024-03-07T03:22:40.842710+0000","last_in_change":"2024-03-07T03:04:39.932769+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":49,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:00:59.846304+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-07T03:06:41.584515+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-07T03:08:48.151945+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-07T03:08:48.827708+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"242c63ee-0417-4ebf-bd71-6239f5d3634f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":468,"up_from":483,"up_thru":1043,"down_at":469,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":3509507411},{"type":"v1","addr":"172.21.15.145:6803","nonce":3509507411}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":3509507411},{"type":"v1","addr":"172.21.15.145:6805","nonce":3509507411}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":3509507411},{"type":"v1","addr":"172.21.15.145:6809","nonce":3509507411}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":3509507411},{"type":"v1","addr":"172.21.15.145:6807","nonce":3509507411}]},"public_addr":"172.21.15.145:6803/3509507411","cluster_addr":"172.21.15.145:6805/3509507411","heartbeat_back_addr":"172.21.15.145:6809/3509507411","heartbeat_front_addr":"172.21.15.145:6807/3509507411","state":["exists","up"]},{"osd":1,"uuid":"3c2818da-5d32-4f7a-8a63-8d7d679961e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":504,"up_from":518,"up_thru":1043,"down_at":505,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":183115113},{"type":"v1","addr":"172.21.15.145:6811","nonce":183115113}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":183115113},{"type":"v1","addr":"172.21.15.145:6813","nonce":183115113}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":183115113},{"type":"v1","addr":"172.21.15.145:6817","nonce":183115113}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":183115113},{"type":"v1","addr":"172.21.15.145:6815","nonce":183115113}]},"public_addr":"172.21.15.145:6811/183115113","cluster_addr":"172.21.15.145:6813/183115113","heartbeat_back_addr":"172.21.15.145:6817/183115113","heartbeat_front_addr":"172.21.15.145:6815/183115113","state":["exists","up"]},{"osd":2,"uuid":"905ec219-dd9c-4f30-bdab-ed78d1405c11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":625,"up_from":636,"up_thru":1043,"down_at":625,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":3198195141},{"type":"v1","addr":"172.21.15.145:6819","nonce":3198195141}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":3198195141},{"type":"v1","addr":"172.21.15.145:6821","nonce":3198195141}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":3198195141},{"type":"v1","addr":"172.21.15.145:6825","nonce":3198195141}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":3198195141},{"type":"v1","addr":"172.21.15.145:6823","nonce":3198195141}]},"public_addr":"172.21.15.145:6819/3198195141","cluster_addr":"172.21.15.145:6821/3198195141","heartbeat_back_addr":"172.21.15.145:6825/3198195141","heartbeat_front_addr":"172.21.15.145:6823/3198195141","state":["exists","up"]},{"osd":3,"uuid":"5f466751-f5a7-4d18-87ab-96251b77b663","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":644,"up_from":648,"up_thru":1043,"down_at":644,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2193661319},{"type":"v1","addr":"172.21.15.145:6827","nonce":2193661319}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2193661319},{"type":"v1","addr":"172.21.15.145:6829","nonce":2193661319}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":2193661319},{"type":"v1","addr":"172.21.15.145:6833","nonce":2193661319}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2193661319},{"type":"v1","addr":"172.21.15.145:6831","nonce":2193661319}]},"public_addr":"172.21.15.145:6827/2193661319","cluster_addr":"172.21.15.145:6829/2193661319","heartbeat_back_addr":"172.21.15.145:6833/2193661319","heartbeat_front_addr":"172.21.15.145:6831/2193661319","state":["exists","up"]},{"osd":4,"uuid":"633f56af-3ff8-4954-97b4-bc8122bb10ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":652,"up_from":655,"up_thru":1043,"down_at":652,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":553416210},{"type":"v1","addr":"172.21.15.189:6801","nonce":553416210}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":553416210},{"type":"v1","addr":"172.21.15.189:6803","nonce":553416210}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":553416210},{"type":"v1","addr":"172.21.15.189:6807","nonce":553416210}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":553416210},{"type":"v1","addr":"172.21.15.189:6805","nonce":553416210}]},"public_addr":"172.21.15.189:6801/553416210","cluster_addr":"172.21.15.189:6803/553416210","heartbeat_back_addr":"172.21.15.189:6807/553416210","heartbeat_front_addr":"172.21.15.189:6805/553416210","state":["exists","up"]},{"osd":5,"uuid":"531d5127-60fe-4669-ab7d-fe4f5e5753cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":1043,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6809","nonce":2096444201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6811","nonce":2096444201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6815","nonce":2096444201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6813","nonce":2096444201}]},"public_addr":"172.21.15.189:6809/2096444201","cluster_addr":"172.21.15.189:6811/2096444201","heartbeat_back_addr":"172.21.15.189:6815/2096444201","heartbeat_front_addr":"172.21.15.189:6813/2096444201","state":["exists","up"]},{"osd":6,"uuid":"8f2b456e-712e-45f1-8d84-0fdc4a6d860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":1043,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6817","nonce":4219543665}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6819","nonce":4219543665}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6823","nonce":4219543665}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6821","nonce":4219543665}]},"public_addr":"172.21.15.189:6817/4219543665","cluster_addr":"172.21.15.189:6819/4219543665","heartbeat_back_addr":"172.21.15.189:6823/4219543665","heartbeat_front_addr":"172.21.15.189:6821/4219543665","state":["exists","up"]},{"osd":7,"uuid":"24dc7eba-4cd4-414d-93d2-598f5f2e655c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":1043,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6825","nonce":452348965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6827","nonce":452348965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6831","nonce":452348965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6829","nonce":452348965}]},"public_addr":"172.21.15.189:6825/452348965","cluster_addr":"172.21.15.189:6827/452348965","heartbeat_back_addr":"172.21.15.189:6831/452348965","heartbeat_front_addr":"172.21.15.189:6829/452348965","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T02:59:11.112897+0000","dead_epoch":468},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:03.501748+0000","dead_epoch":504},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:56.635004+0000","dead_epoch":625},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:01:50.372735+0000","dead_epoch":644},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:38.150721+0000","dead_epoch":652},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:25.965239+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:14.345171+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:03.479901+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.145:0/3252190224":"2024-03-07T04:20:03.607131+0000","172.21.15.145:6835/3433351665":"2024-03-08T03:08:57.328270+0000","172.21.15.145:0/2957778655":"2024-03-08T03:15:02.496018+0000","172.21.15.145:6800/1684821814":"2024-03-08T03:13:03.366616+0000","172.21.15.145:6801/1684821814":"2024-03-08T03:13:03.366616+0000","172.21.15.145:0/2414597158":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/966499613":"2024-03-08T03:15:02.496018+0000","172.21.15.145:0/1659536498":"2024-03-07T04:29:26.868302+0000","172.21.15.145:6800/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.189:0/692984461":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/2696605014":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/2417578797":"2024-03-08T03:15:02.496018+0000","172.21.15.145:0/3395344828":"2024-03-07T04:20:04.550850+0000","172.21.15.189:0/1606183729":"2024-03-08T03:13:57.642513+0000","172.21.15.189:6834/4006065880":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/131277059":"2024-03-08T02:56:14.659813+0000","172.21.15.189:6835/4006065880":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/321164949":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/4175990096":"2024-03-08T03:15:02.496018+0000","172.21.15.145:0/1139507458":"2024-03-08T03:13:03.366616+0000","172.21.15.145:6800/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:6800/4112237851":"2024-03-08T03:15:02.496018+0000","172.21.15.145:0/1978862755":"2024-03-08T03:06:02.331013+0000","172.21.15.189:0/2261244840":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/3299433409":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3488231986":"2024-03-08T03:06:02.331013+0000","172.21.15.145:6834/3433351665":"2024-03-08T03:08:57.328270+0000","172.21.15.145:0/2236140381":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/1420640097":"2024-03-08T03:13:03.366616+0000","172.21.15.145:6801/4112237851":"2024-03-08T03:15:02.496018+0000","172.21.15.189:0/1082708129":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/3741881182":"2024-03-08T03:15:02.496018+0000","172.21.15.145:6801/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.189:0/3157155685":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/4002816623":"2024-03-08T03:13:03.366616+0000","172.21.15.145:6800/2891889777":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/2939737297":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/2891889777":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3349297031":"2024-03-08T03:13:03.366616+0000","172.21.15.145:0/3852118874":"2024-03-08T03:06:02.331013+0000","172.21.15.189:0/3779336862":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/89527742":"2024-03-08T03:15:02.496018+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":14,"snaps":[{"begin":5,"length":4}]},{"pool":16,"snaps":[{"begin":35,"length":3},{"begin":45,"length":3}]},{"pool":17,"snaps":[{"begin":17,"length":2},{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":49,"snaps":[{"begin":14,"length":3}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:49:22.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:21 smithi145 bash[79878]: audit 2024-03-07T03:49:21.296836+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.145:0/4035746655' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:49:22.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:21 smithi145 bash[82329]: audit 2024-03-07T03:49:21.296836+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.145:0/4035746655' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:49:22.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:21 smithi189 bash[66969]: audit 2024-03-07T03:49:21.296836+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.145:0/4035746655' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:49:22.132 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-07T03:49:22.163 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:49:22.588 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:49:22 smithi189 bash[59814]: ts=2024-03-07T03:49:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:49:23.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:22 smithi145 bash[79878]: cluster 2024-03-07T03:49:21.670579+0000 mgr.y (mgr.34107) 1125 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:23.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:49:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:49:22] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.43.0" 2024-03-07T03:49:23.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:22 smithi145 bash[82329]: cluster 2024-03-07T03:49:21.670579+0000 mgr.y (mgr.34107) 1125 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:23.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:22 smithi189 bash[66969]: cluster 2024-03-07T03:49:21.670579+0000 mgr.y (mgr.34107) 1125 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:25.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:24 smithi145 bash[79878]: cluster 2024-03-07T03:49:23.671488+0000 mgr.y (mgr.34107) 1126 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:25.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:24 smithi145 bash[82329]: cluster 2024-03-07T03:49:23.671488+0000 mgr.y (mgr.34107) 1126 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:25.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:24 smithi189 bash[66969]: cluster 2024-03-07T03:49:23.671488+0000 mgr.y (mgr.34107) 1126 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:26.903 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:49:27.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:26 smithi145 bash[79878]: cluster 2024-03-07T03:49:25.672254+0000 mgr.y (mgr.34107) 1127 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:27.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:26 smithi145 bash[82329]: cluster 2024-03-07T03:49:25.672254+0000 mgr.y (mgr.34107) 1127 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:27.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:26 smithi189 bash[66969]: cluster 2024-03-07T03:49:25.672254+0000 mgr.y (mgr.34107) 1127 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:29.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:28 smithi145 bash[79878]: cluster 2024-03-07T03:49:27.673166+0000 mgr.y (mgr.34107) 1128 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:29.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:28 smithi145 bash[82329]: cluster 2024-03-07T03:49:27.673166+0000 mgr.y (mgr.34107) 1128 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:29.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:28 smithi189 bash[66969]: cluster 2024-03-07T03:49:27.673166+0000 mgr.y (mgr.34107) 1128 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:30.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:29 smithi145 bash[79878]: audit 2024-03-07T03:49:29.089945+0000 mon.a (mon.0) 1959 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:49:30.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:29 smithi145 bash[79878]: audit 2024-03-07T03:49:29.614751+0000 mon.a (mon.0) 1960 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:30.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:29 smithi145 bash[79878]: audit 2024-03-07T03:49:29.620543+0000 mon.a (mon.0) 1961 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:30.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:29 smithi145 bash[82329]: audit 2024-03-07T03:49:29.089945+0000 mon.a (mon.0) 1959 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:49:30.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:29 smithi145 bash[82329]: audit 2024-03-07T03:49:29.614751+0000 mon.a (mon.0) 1960 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:30.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:29 smithi145 bash[82329]: audit 2024-03-07T03:49:29.620543+0000 mon.a (mon.0) 1961 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:30.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:29 smithi189 bash[66969]: audit 2024-03-07T03:49:29.089945+0000 mon.a (mon.0) 1959 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:49:30.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:29 smithi189 bash[66969]: audit 2024-03-07T03:49:29.614751+0000 mon.a (mon.0) 1960 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:30.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:29 smithi189 bash[66969]: audit 2024-03-07T03:49:29.620543+0000 mon.a (mon.0) 1961 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:30.221 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:49:30.221 INFO:teuthology.orchestra.run.smithi145.stdout:{"epoch":1101,"fsid":"8947a0f8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:55:42.039360+0000","modified":"2024-03-07T03:49:03.137663+0000","last_up_change":"2024-03-07T03:22:40.842710+0000","last_in_change":"2024-03-07T03:04:39.932769+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":49,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:00:59.846304+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-07T03:06:41.584515+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-07T03:08:48.151945+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-07T03:08:48.827708+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"242c63ee-0417-4ebf-bd71-6239f5d3634f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":468,"up_from":483,"up_thru":1043,"down_at":469,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6802","nonce":3509507411},{"type":"v1","addr":"172.21.15.145:6803","nonce":3509507411}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6804","nonce":3509507411},{"type":"v1","addr":"172.21.15.145:6805","nonce":3509507411}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6808","nonce":3509507411},{"type":"v1","addr":"172.21.15.145:6809","nonce":3509507411}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6806","nonce":3509507411},{"type":"v1","addr":"172.21.15.145:6807","nonce":3509507411}]},"public_addr":"172.21.15.145:6803/3509507411","cluster_addr":"172.21.15.145:6805/3509507411","heartbeat_back_addr":"172.21.15.145:6809/3509507411","heartbeat_front_addr":"172.21.15.145:6807/3509507411","state":["exists","up"]},{"osd":1,"uuid":"3c2818da-5d32-4f7a-8a63-8d7d679961e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":504,"up_from":518,"up_thru":1043,"down_at":505,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6810","nonce":183115113},{"type":"v1","addr":"172.21.15.145:6811","nonce":183115113}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6812","nonce":183115113},{"type":"v1","addr":"172.21.15.145:6813","nonce":183115113}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6816","nonce":183115113},{"type":"v1","addr":"172.21.15.145:6817","nonce":183115113}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6814","nonce":183115113},{"type":"v1","addr":"172.21.15.145:6815","nonce":183115113}]},"public_addr":"172.21.15.145:6811/183115113","cluster_addr":"172.21.15.145:6813/183115113","heartbeat_back_addr":"172.21.15.145:6817/183115113","heartbeat_front_addr":"172.21.15.145:6815/183115113","state":["exists","up"]},{"osd":2,"uuid":"905ec219-dd9c-4f30-bdab-ed78d1405c11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":625,"up_from":636,"up_thru":1043,"down_at":625,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6818","nonce":3198195141},{"type":"v1","addr":"172.21.15.145:6819","nonce":3198195141}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6820","nonce":3198195141},{"type":"v1","addr":"172.21.15.145:6821","nonce":3198195141}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6824","nonce":3198195141},{"type":"v1","addr":"172.21.15.145:6825","nonce":3198195141}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6822","nonce":3198195141},{"type":"v1","addr":"172.21.15.145:6823","nonce":3198195141}]},"public_addr":"172.21.15.145:6819/3198195141","cluster_addr":"172.21.15.145:6821/3198195141","heartbeat_back_addr":"172.21.15.145:6825/3198195141","heartbeat_front_addr":"172.21.15.145:6823/3198195141","state":["exists","up"]},{"osd":3,"uuid":"5f466751-f5a7-4d18-87ab-96251b77b663","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":644,"up_from":648,"up_thru":1043,"down_at":644,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6826","nonce":2193661319},{"type":"v1","addr":"172.21.15.145:6827","nonce":2193661319}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6828","nonce":2193661319},{"type":"v1","addr":"172.21.15.145:6829","nonce":2193661319}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6832","nonce":2193661319},{"type":"v1","addr":"172.21.15.145:6833","nonce":2193661319}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.145:6830","nonce":2193661319},{"type":"v1","addr":"172.21.15.145:6831","nonce":2193661319}]},"public_addr":"172.21.15.145:6827/2193661319","cluster_addr":"172.21.15.145:6829/2193661319","heartbeat_back_addr":"172.21.15.145:6833/2193661319","heartbeat_front_addr":"172.21.15.145:6831/2193661319","state":["exists","up"]},{"osd":4,"uuid":"633f56af-3ff8-4954-97b4-bc8122bb10ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":652,"up_from":655,"up_thru":1043,"down_at":652,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":553416210},{"type":"v1","addr":"172.21.15.189:6801","nonce":553416210}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":553416210},{"type":"v1","addr":"172.21.15.189:6803","nonce":553416210}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":553416210},{"type":"v1","addr":"172.21.15.189:6807","nonce":553416210}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":553416210},{"type":"v1","addr":"172.21.15.189:6805","nonce":553416210}]},"public_addr":"172.21.15.189:6801/553416210","cluster_addr":"172.21.15.189:6803/553416210","heartbeat_back_addr":"172.21.15.189:6807/553416210","heartbeat_front_addr":"172.21.15.189:6805/553416210","state":["exists","up"]},{"osd":5,"uuid":"531d5127-60fe-4669-ab7d-fe4f5e5753cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":1043,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6809","nonce":2096444201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6811","nonce":2096444201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6815","nonce":2096444201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":2096444201},{"type":"v1","addr":"172.21.15.189:6813","nonce":2096444201}]},"public_addr":"172.21.15.189:6809/2096444201","cluster_addr":"172.21.15.189:6811/2096444201","heartbeat_back_addr":"172.21.15.189:6815/2096444201","heartbeat_front_addr":"172.21.15.189:6813/2096444201","state":["exists","up"]},{"osd":6,"uuid":"8f2b456e-712e-45f1-8d84-0fdc4a6d860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":1043,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6817","nonce":4219543665}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6819","nonce":4219543665}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6823","nonce":4219543665}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":4219543665},{"type":"v1","addr":"172.21.15.189:6821","nonce":4219543665}]},"public_addr":"172.21.15.189:6817/4219543665","cluster_addr":"172.21.15.189:6819/4219543665","heartbeat_back_addr":"172.21.15.189:6823/4219543665","heartbeat_front_addr":"172.21.15.189:6821/4219543665","state":["exists","up"]},{"osd":7,"uuid":"24dc7eba-4cd4-414d-93d2-598f5f2e655c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":1043,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6825","nonce":452348965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6827","nonce":452348965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6831","nonce":452348965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":452348965},{"type":"v1","addr":"172.21.15.189:6829","nonce":452348965}]},"public_addr":"172.21.15.189:6825/452348965","cluster_addr":"172.21.15.189:6827/452348965","heartbeat_back_addr":"172.21.15.189:6831/452348965","heartbeat_front_addr":"172.21.15.189:6829/452348965","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T02:59:11.112897+0000","dead_epoch":468},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:03.501748+0000","dead_epoch":504},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:00:56.635004+0000","dead_epoch":625},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:01:50.372735+0000","dead_epoch":644},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:38.150721+0000","dead_epoch":652},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:25.965239+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:14.345171+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:03.479901+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.145:0/3252190224":"2024-03-07T04:20:03.607131+0000","172.21.15.145:6835/3433351665":"2024-03-08T03:08:57.328270+0000","172.21.15.145:0/2957778655":"2024-03-08T03:15:02.496018+0000","172.21.15.145:6800/1684821814":"2024-03-08T03:13:03.366616+0000","172.21.15.145:6801/1684821814":"2024-03-08T03:13:03.366616+0000","172.21.15.145:0/2414597158":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/966499613":"2024-03-08T03:15:02.496018+0000","172.21.15.145:0/1659536498":"2024-03-07T04:29:26.868302+0000","172.21.15.145:6800/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.189:0/692984461":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/2696605014":"2024-03-08T02:56:47.212702+0000","172.21.15.145:0/2417578797":"2024-03-08T03:15:02.496018+0000","172.21.15.145:0/3395344828":"2024-03-07T04:20:04.550850+0000","172.21.15.189:0/1606183729":"2024-03-08T03:13:57.642513+0000","172.21.15.189:6834/4006065880":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/131277059":"2024-03-08T02:56:14.659813+0000","172.21.15.189:6835/4006065880":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/321164949":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/4175990096":"2024-03-08T03:15:02.496018+0000","172.21.15.145:0/1139507458":"2024-03-08T03:13:03.366616+0000","172.21.15.145:6800/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/444730212":"2024-03-08T02:56:14.659813+0000","172.21.15.145:6800/4112237851":"2024-03-08T03:15:02.496018+0000","172.21.15.145:0/1978862755":"2024-03-08T03:06:02.331013+0000","172.21.15.189:0/2261244840":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/3299433409":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3488231986":"2024-03-08T03:06:02.331013+0000","172.21.15.145:6834/3433351665":"2024-03-08T03:08:57.328270+0000","172.21.15.145:0/2236140381":"2024-03-08T02:56:14.659813+0000","172.21.15.145:0/1420640097":"2024-03-08T03:13:03.366616+0000","172.21.15.145:6801/4112237851":"2024-03-08T03:15:02.496018+0000","172.21.15.189:0/1082708129":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/3741881182":"2024-03-08T03:15:02.496018+0000","172.21.15.145:6801/1256451856":"2024-03-08T02:56:47.212702+0000","172.21.15.189:0/3157155685":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/4002816623":"2024-03-08T03:13:03.366616+0000","172.21.15.145:6800/2891889777":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/2939737297":"2024-03-08T02:56:47.212702+0000","172.21.15.145:6801/2891889777":"2024-03-08T03:06:02.331013+0000","172.21.15.145:0/3349297031":"2024-03-08T03:13:03.366616+0000","172.21.15.145:0/3852118874":"2024-03-08T03:06:02.331013+0000","172.21.15.189:0/3779336862":"2024-03-08T03:13:57.642513+0000","172.21.15.145:0/89527742":"2024-03-08T03:15:02.496018+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":14,"snaps":[{"begin":5,"length":4}]},{"pool":16,"snaps":[{"begin":35,"length":3},{"begin":45,"length":3}]},{"pool":17,"snaps":[{"begin":17,"length":2},{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":49,"snaps":[{"begin":14,"length":3}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:49:31.006 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-07T03:49:31.006 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-07T03:49:31.007 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-07T03:49:31.007 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-07T03:49:31.007 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-07T03:49:31.008 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-07T03:49:31.008 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-07T03:49:31.009 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-07T03:49:31.028 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:30 smithi145 bash[82329]: cluster 2024-03-07T03:49:29.674115+0000 mgr.y (mgr.34107) 1129 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:31.029 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:30 smithi145 bash[82329]: audit 2024-03-07T03:49:30.222246+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.145:0/1961927996' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:49:31.029 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:30 smithi145 bash[79878]: cluster 2024-03-07T03:49:29.674115+0000 mgr.y (mgr.34107) 1129 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:31.029 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:30 smithi145 bash[79878]: audit 2024-03-07T03:49:30.222246+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.145:0/1961927996' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:49:31.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:30 smithi189 bash[66969]: cluster 2024-03-07T03:49:29.674115+0000 mgr.y (mgr.34107) 1129 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:31.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:30 smithi189 bash[66969]: audit 2024-03-07T03:49:30.222246+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.145:0/1961927996' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:49:32.588 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:49:32 smithi189 bash[59814]: ts=2024-03-07T03:49:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:49:33.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:49:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:49:32] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2024-03-07T03:49:33.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:32 smithi145 bash[79878]: cluster 2024-03-07T03:49:31.674975+0000 mgr.y (mgr.34107) 1130 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:33.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:32 smithi145 bash[79878]: audit 2024-03-07T03:49:32.684314+0000 mon.a (mon.0) 1962 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:33.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:32 smithi145 bash[82329]: cluster 2024-03-07T03:49:31.674975+0000 mgr.y (mgr.34107) 1130 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:33.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:32 smithi145 bash[82329]: audit 2024-03-07T03:49:32.684314+0000 mon.a (mon.0) 1962 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:33.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:32 smithi189 bash[66969]: cluster 2024-03-07T03:49:31.674975+0000 mgr.y (mgr.34107) 1130 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:33.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:32 smithi189 bash[66969]: audit 2024-03-07T03:49:32.684314+0000 mon.a (mon.0) 1962 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:35.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:34 smithi145 bash[79878]: cluster 2024-03-07T03:49:33.675969+0000 mgr.y (mgr.34107) 1131 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:35.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:34 smithi145 bash[82329]: cluster 2024-03-07T03:49:33.675969+0000 mgr.y (mgr.34107) 1131 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:35.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:34 smithi189 bash[66969]: cluster 2024-03-07T03:49:33.675969+0000 mgr.y (mgr.34107) 1131 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:35.563 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:49:35.563 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:49:35.564 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:49:35.564 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:49:35.564 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:49:35.565 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:49:35.565 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:49:35.567 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:49:36.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[82329]: audit 2024-03-07T03:49:35.391637+0000 mon.a (mon.0) 1963 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[82329]: audit 2024-03-07T03:49:35.402947+0000 mon.a (mon.0) 1964 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[82329]: audit 2024-03-07T03:49:35.616762+0000 mon.a (mon.0) 1965 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[82329]: audit 2024-03-07T03:49:35.621749+0000 mon.a (mon.0) 1966 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[82329]: cluster 2024-03-07T03:49:35.676489+0000 mgr.y (mgr.34107) 1132 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:36.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[82329]: audit 2024-03-07T03:49:36.147580+0000 mon.a (mon.0) 1967 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:49:36.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[82329]: audit 2024-03-07T03:49:36.149351+0000 mon.a (mon.0) 1968 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:49:36.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[82329]: audit 2024-03-07T03:49:36.159488+0000 mon.a (mon.0) 1969 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[79878]: audit 2024-03-07T03:49:35.391637+0000 mon.a (mon.0) 1963 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[79878]: audit 2024-03-07T03:49:35.402947+0000 mon.a (mon.0) 1964 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[79878]: audit 2024-03-07T03:49:35.616762+0000 mon.a (mon.0) 1965 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[79878]: audit 2024-03-07T03:49:35.621749+0000 mon.a (mon.0) 1966 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[79878]: cluster 2024-03-07T03:49:35.676489+0000 mgr.y (mgr.34107) 1132 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:36.820 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[79878]: audit 2024-03-07T03:49:36.147580+0000 mon.a (mon.0) 1967 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:49:36.820 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[79878]: audit 2024-03-07T03:49:36.149351+0000 mon.a (mon.0) 1968 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:49:36.820 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:36 smithi145 bash[79878]: audit 2024-03-07T03:49:36.159488+0000 mon.a (mon.0) 1969 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.838 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:36 smithi189 bash[66969]: audit 2024-03-07T03:49:35.391637+0000 mon.a (mon.0) 1963 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:36 smithi189 bash[66969]: audit 2024-03-07T03:49:35.402947+0000 mon.a (mon.0) 1964 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:36 smithi189 bash[66969]: audit 2024-03-07T03:49:35.616762+0000 mon.a (mon.0) 1965 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:36 smithi189 bash[66969]: audit 2024-03-07T03:49:35.621749+0000 mon.a (mon.0) 1966 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:36.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:36 smithi189 bash[66969]: cluster 2024-03-07T03:49:35.676489+0000 mgr.y (mgr.34107) 1132 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:36.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:36 smithi189 bash[66969]: audit 2024-03-07T03:49:36.147580+0000 mon.a (mon.0) 1967 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:49:36.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:36 smithi189 bash[66969]: audit 2024-03-07T03:49:36.149351+0000 mon.a (mon.0) 1968 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:49:36.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:36 smithi189 bash[66969]: audit 2024-03-07T03:49:36.159488+0000 mon.a (mon.0) 1969 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:49:39.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:38 smithi145 bash[79878]: cluster 2024-03-07T03:49:37.677394+0000 mgr.y (mgr.34107) 1133 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:39.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:38 smithi145 bash[82329]: cluster 2024-03-07T03:49:37.677394+0000 mgr.y (mgr.34107) 1133 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:39.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:38 smithi189 bash[66969]: cluster 2024-03-07T03:49:37.677394+0000 mgr.y (mgr.34107) 1133 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:41.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:40 smithi189 bash[66969]: cluster 2024-03-07T03:49:39.677963+0000 mgr.y (mgr.34107) 1134 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:41.170 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:40 smithi145 bash[79878]: cluster 2024-03-07T03:49:39.677963+0000 mgr.y (mgr.34107) 1134 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:41.170 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:40 smithi145 bash[82329]: cluster 2024-03-07T03:49:39.677963+0000 mgr.y (mgr.34107) 1134 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:42.588 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:49:42 smithi189 bash[59814]: ts=2024-03-07T03:49:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:49:42.818 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:49:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:49:42] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2024-03-07T03:49:42.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:42 smithi145 bash[82329]: cluster 2024-03-07T03:49:41.678836+0000 mgr.y (mgr.34107) 1135 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:42.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:42 smithi145 bash[79878]: cluster 2024-03-07T03:49:41.678836+0000 mgr.y (mgr.34107) 1135 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:43.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:42 smithi189 bash[66969]: cluster 2024-03-07T03:49:41.678836+0000 mgr.y (mgr.34107) 1135 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:45.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:44 smithi145 bash[79878]: cluster 2024-03-07T03:49:43.679536+0000 mgr.y (mgr.34107) 1136 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:45.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:44 smithi145 bash[82329]: cluster 2024-03-07T03:49:43.679536+0000 mgr.y (mgr.34107) 1136 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:45.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:44 smithi189 bash[66969]: cluster 2024-03-07T03:49:43.679536+0000 mgr.y (mgr.34107) 1136 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:47.088 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:46 smithi145 bash[79878]: cluster 2024-03-07T03:49:45.680369+0000 mgr.y (mgr.34107) 1137 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:47.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:46 smithi189 bash[66969]: cluster 2024-03-07T03:49:45.680369+0000 mgr.y (mgr.34107) 1137 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:47.089 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:46 smithi145 bash[82329]: cluster 2024-03-07T03:49:45.680369+0000 mgr.y (mgr.34107) 1137 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:48.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:47 smithi189 bash[66969]: audit 2024-03-07T03:49:47.684628+0000 mon.a (mon.0) 1970 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:48.091 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:47 smithi145 bash[79878]: audit 2024-03-07T03:49:47.684628+0000 mon.a (mon.0) 1970 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:48.091 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:47 smithi145 bash[82329]: audit 2024-03-07T03:49:47.684628+0000 mon.a (mon.0) 1970 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:49.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:48 smithi145 bash[79878]: cluster 2024-03-07T03:49:47.681349+0000 mgr.y (mgr.34107) 1138 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:49.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:48 smithi145 bash[82329]: cluster 2024-03-07T03:49:47.681349+0000 mgr.y (mgr.34107) 1138 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:49.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:48 smithi189 bash[66969]: cluster 2024-03-07T03:49:47.681349+0000 mgr.y (mgr.34107) 1138 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:50.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:50 smithi145 bash[79878]: cluster 2024-03-07T03:49:49.681932+0000 mgr.y (mgr.34107) 1139 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:50.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:50 smithi145 bash[82329]: cluster 2024-03-07T03:49:49.681932+0000 mgr.y (mgr.34107) 1139 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:51.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:50 smithi189 bash[66969]: cluster 2024-03-07T03:49:49.681932+0000 mgr.y (mgr.34107) 1139 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:52.588 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:49:52 smithi189 bash[59814]: ts=2024-03-07T03:49:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:49:52.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:52 smithi145 bash[79878]: cluster 2024-03-07T03:49:51.682409+0000 mgr.y (mgr.34107) 1140 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:52.818 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:49:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:49:52] "GET /metrics HTTP/1.1" 200 34153 "" "Prometheus/2.43.0" 2024-03-07T03:49:52.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:52 smithi145 bash[82329]: cluster 2024-03-07T03:49:51.682409+0000 mgr.y (mgr.34107) 1140 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:53.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:52 smithi189 bash[66969]: cluster 2024-03-07T03:49:51.682409+0000 mgr.y (mgr.34107) 1140 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:53.977 INFO:teuthology.orchestra.run.smithi145.stdout:2783138808141 2024-03-07T03:49:53.977 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-07T03:49:54.237 INFO:teuthology.orchestra.run.smithi145.stdout:201863463461 2024-03-07T03:49:54.237 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-07T03:49:54.526 INFO:teuthology.orchestra.run.smithi145.stdout:2813203579208 2024-03-07T03:49:54.526 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-07T03:49:54.818 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:54 smithi145 bash[79878]: cluster 2024-03-07T03:49:53.683322+0000 mgr.y (mgr.34107) 1141 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:54.818 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:54 smithi145 bash[82329]: cluster 2024-03-07T03:49:53.683322+0000 mgr.y (mgr.34107) 1141 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:55.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:54 smithi189 bash[66969]: cluster 2024-03-07T03:49:53.683322+0000 mgr.y (mgr.34107) 1141 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:56.128 INFO:teuthology.orchestra.run.smithi145.stdout:2731599200596 2024-03-07T03:49:56.129 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-07T03:49:56.311 INFO:teuthology.orchestra.run.smithi145.stdout:167503725103 2024-03-07T03:49:56.312 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-07T03:49:57.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:56 smithi145 bash[79878]: cluster 2024-03-07T03:49:55.684255+0000 mgr.y (mgr.34107) 1142 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:57.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:56 smithi145 bash[82329]: cluster 2024-03-07T03:49:55.684255+0000 mgr.y (mgr.34107) 1142 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:57.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:56 smithi189 bash[66969]: cluster 2024-03-07T03:49:55.684255+0000 mgr.y (mgr.34107) 1142 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:57.317 INFO:teuthology.orchestra.run.smithi145.stdout:2224793059693 2024-03-07T03:49:57.318 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-07T03:49:57.798 INFO:teuthology.orchestra.run.smithi145.stdout:2074469204340 2024-03-07T03:49:57.798 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-07T03:49:57.843 INFO:teuthology.orchestra.run.smithi145.stdout:236223201820 2024-03-07T03:49:57.844 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-07T03:49:59.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:49:58 smithi145 bash[79878]: cluster 2024-03-07T03:49:57.685200+0000 mgr.y (mgr.34107) 1143 : cluster 0 pgmap v1869: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:59.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:49:58 smithi145 bash[82329]: cluster 2024-03-07T03:49:57.685200+0000 mgr.y (mgr.34107) 1143 : cluster 0 pgmap v1869: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:49:59.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:49:58 smithi189 bash[66969]: cluster 2024-03-07T03:49:57.685200+0000 mgr.y (mgr.34107) 1143 : cluster 0 pgmap v1869: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:00.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:00 smithi145 bash[79878]: debug 2024-03-07T03:49:59.996+0000 7fdb49eda700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:50:00.746 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:50:00.746 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:50:00.747 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:50:00.747 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:50:00.747 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:50:01.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:00 smithi145 bash[79878]: cluster 2024-03-07T03:49:59.686124+0000 mgr.y (mgr.34107) 1144 : cluster 0 pgmap v1870: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:01.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:00 smithi145 bash[79878]: cluster 2024-03-07T03:50:00.000118+0000 mon.a (mon.0) 1971 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:50:01.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:00 smithi145 bash[82329]: cluster 2024-03-07T03:49:59.686124+0000 mgr.y (mgr.34107) 1144 : cluster 0 pgmap v1870: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:01.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:00 smithi145 bash[82329]: cluster 2024-03-07T03:50:00.000118+0000 mon.a (mon.0) 1971 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:50:01.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:00 smithi189 bash[66969]: cluster 2024-03-07T03:49:59.686124+0000 mgr.y (mgr.34107) 1144 : cluster 0 pgmap v1870: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:01.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:00 smithi189 bash[66969]: cluster 2024-03-07T03:50:00.000118+0000 mon.a (mon.0) 1971 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:50:01.764 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:50:01.765 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:50:01.765 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:50:02.589 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:50:02 smithi189 bash[59814]: ts=2024-03-07T03:50:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:50:03.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:50:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:50:02] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.43.0" 2024-03-07T03:50:03.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:02 smithi145 bash[79878]: cluster 2024-03-07T03:50:01.686725+0000 mgr.y (mgr.34107) 1145 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:03.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:02 smithi145 bash[79878]: audit 2024-03-07T03:50:02.685002+0000 mon.a (mon.0) 1972 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:03.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:02 smithi145 bash[82329]: cluster 2024-03-07T03:50:01.686725+0000 mgr.y (mgr.34107) 1145 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:03.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:02 smithi145 bash[82329]: audit 2024-03-07T03:50:02.685002+0000 mon.a (mon.0) 1972 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:03.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:02 smithi189 bash[66969]: cluster 2024-03-07T03:50:01.686725+0000 mgr.y (mgr.34107) 1145 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:03.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:02 smithi189 bash[66969]: audit 2024-03-07T03:50:02.685002+0000 mon.a (mon.0) 1972 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:05.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:04 smithi145 bash[79878]: cluster 2024-03-07T03:50:03.687601+0000 mgr.y (mgr.34107) 1146 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:05.068 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:04 smithi145 bash[82329]: cluster 2024-03-07T03:50:03.687601+0000 mgr.y (mgr.34107) 1146 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:05.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:04 smithi189 bash[66969]: cluster 2024-03-07T03:50:03.687601+0000 mgr.y (mgr.34107) 1146 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:06.922 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:06 smithi145 bash[79878]: cluster 2024-03-07T03:50:05.688338+0000 mgr.y (mgr.34107) 1147 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:06.922 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:06 smithi145 bash[82329]: cluster 2024-03-07T03:50:05.688338+0000 mgr.y (mgr.34107) 1147 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:07.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:06 smithi189 bash[66969]: cluster 2024-03-07T03:50:05.688338+0000 mgr.y (mgr.34107) 1147 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:09.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:08 smithi145 bash[79878]: cluster 2024-03-07T03:50:07.688962+0000 mgr.y (mgr.34107) 1148 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:09.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:08 smithi145 bash[82329]: cluster 2024-03-07T03:50:07.688962+0000 mgr.y (mgr.34107) 1148 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:09.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:08 smithi189 bash[66969]: cluster 2024-03-07T03:50:07.688962+0000 mgr.y (mgr.34107) 1148 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:11.079 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:10 smithi145 bash[79878]: cluster 2024-03-07T03:50:09.689500+0000 mgr.y (mgr.34107) 1149 : cluster 0 pgmap v1875: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:11.080 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:10 smithi145 bash[82329]: cluster 2024-03-07T03:50:09.689500+0000 mgr.y (mgr.34107) 1149 : cluster 0 pgmap v1875: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:11.088 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:10 smithi189 bash[66969]: cluster 2024-03-07T03:50:09.689500+0000 mgr.y (mgr.34107) 1149 : cluster 0 pgmap v1875: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:12.589 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:50:12 smithi189 bash[59814]: ts=2024-03-07T03:50:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:50:13.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:12 smithi145 bash[79878]: cluster 2024-03-07T03:50:11.690432+0000 mgr.y (mgr.34107) 1150 : cluster 0 pgmap v1876: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:13.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:50:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:50:12] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.43.0" 2024-03-07T03:50:13.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:12 smithi145 bash[82329]: cluster 2024-03-07T03:50:11.690432+0000 mgr.y (mgr.34107) 1150 : cluster 0 pgmap v1876: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:13.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:12 smithi189 bash[66969]: cluster 2024-03-07T03:50:11.690432+0000 mgr.y (mgr.34107) 1150 : cluster 0 pgmap v1876: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:15.318 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:15 smithi145 bash[79878]: cluster 2024-03-07T03:50:13.691265+0000 mgr.y (mgr.34107) 1151 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:15.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:15 smithi145 bash[82329]: cluster 2024-03-07T03:50:13.691265+0000 mgr.y (mgr.34107) 1151 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:15.531 INFO:teuthology.orchestra.run.smithi145.stdout:2813203579212 2024-03-07T03:50:15.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:15 smithi189 bash[66969]: cluster 2024-03-07T03:50:13.691265+0000 mgr.y (mgr.34107) 1151 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:16.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:16 smithi145 bash[79878]: audit 2024-03-07T03:50:15.534228+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.145:0/3959254045' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:50:16.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:16 smithi145 bash[82329]: audit 2024-03-07T03:50:15.534228+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.145:0/3959254045' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:50:16.588 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:16 smithi189 bash[66969]: audit 2024-03-07T03:50:15.534228+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.145:0/3959254045' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:50:17.568 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:17 smithi145 bash[79878]: cluster 2024-03-07T03:50:15.691846+0000 mgr.y (mgr.34107) 1152 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:17.569 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:17 smithi145 bash[82329]: cluster 2024-03-07T03:50:15.691846+0000 mgr.y (mgr.34107) 1152 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:17.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:17 smithi189 bash[66969]: cluster 2024-03-07T03:50:15.691846+0000 mgr.y (mgr.34107) 1152 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:17.986 INFO:teuthology.orchestra.run.smithi145.stdout:2731599200600 2024-03-07T03:50:18.098 INFO:teuthology.orchestra.run.smithi145.stdout:167503725108 2024-03-07T03:50:18.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:18 smithi189 bash[66969]: audit 2024-03-07T03:50:17.684587+0000 mon.a (mon.0) 1973 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:18.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:18 smithi189 bash[66969]: cluster 2024-03-07T03:50:17.692423+0000 mgr.y (mgr.34107) 1153 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:18.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:18 smithi189 bash[66969]: audit 2024-03-07T03:50:17.989102+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.145:0/2889003678' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:50:18.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:18 smithi189 bash[66969]: audit 2024-03-07T03:50:18.101568+0000 mon.a (mon.0) 1974 : audit 0 from='client.? 172.21.15.145:0/2588774686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:50:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:18 smithi145 bash[79878]: audit 2024-03-07T03:50:17.684587+0000 mon.a (mon.0) 1973 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:18 smithi145 bash[79878]: cluster 2024-03-07T03:50:17.692423+0000 mgr.y (mgr.34107) 1153 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:18 smithi145 bash[79878]: audit 2024-03-07T03:50:17.989102+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.145:0/2889003678' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:50:18.624 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:18 smithi145 bash[79878]: audit 2024-03-07T03:50:18.101568+0000 mon.a (mon.0) 1974 : audit 0 from='client.? 172.21.15.145:0/2588774686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:50:18.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:18 smithi145 bash[82329]: audit 2024-03-07T03:50:17.684587+0000 mon.a (mon.0) 1973 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:18.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:18 smithi145 bash[82329]: cluster 2024-03-07T03:50:17.692423+0000 mgr.y (mgr.34107) 1153 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:18.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:18 smithi145 bash[82329]: audit 2024-03-07T03:50:17.989102+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.145:0/2889003678' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:50:18.624 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:18 smithi145 bash[82329]: audit 2024-03-07T03:50:18.101568+0000 mon.a (mon.0) 1974 : audit 0 from='client.? 172.21.15.145:0/2588774686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:50:19.332 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2813203579208 got 2813203579212 for osd.4 2024-03-07T03:50:19.332 DEBUG:teuthology.parallel:result is None 2024-03-07T03:50:19.572 INFO:teuthology.orchestra.run.smithi145.stdout:2783138808147 2024-03-07T03:50:19.852 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:19 smithi145 bash[79878]: audit 2024-03-07T03:50:19.575242+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.145:0/3318116567' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:50:19.852 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:19 smithi145 bash[82329]: audit 2024-03-07T03:50:19.575242+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.145:0/3318116567' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:50:20.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:19 smithi189 bash[66969]: audit 2024-03-07T03:50:19.575242+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.145:0/3318116567' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:50:20.228 INFO:teuthology.orchestra.run.smithi145.stdout:236223201824 2024-03-07T03:50:20.749 INFO:teuthology.orchestra.run.smithi145.stdout:201863463467 2024-03-07T03:50:20.891 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:20 smithi145 bash[79878]: cluster 2024-03-07T03:50:19.693223+0000 mgr.y (mgr.34107) 1154 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:20.891 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:20 smithi145 bash[79878]: audit 2024-03-07T03:50:20.231442+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.145:0/2623394262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:50:20.891 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:20 smithi145 bash[82329]: cluster 2024-03-07T03:50:19.693223+0000 mgr.y (mgr.34107) 1154 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:20.891 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:20 smithi145 bash[82329]: audit 2024-03-07T03:50:20.231442+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.145:0/2623394262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:50:21.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:20 smithi189 bash[66969]: cluster 2024-03-07T03:50:19.693223+0000 mgr.y (mgr.34107) 1154 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:21.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:20 smithi189 bash[66969]: audit 2024-03-07T03:50:20.231442+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.145:0/2623394262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:50:22.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:21 smithi145 bash[79878]: audit 2024-03-07T03:50:20.752559+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.145:0/3612969278' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:50:22.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:21 smithi145 bash[82329]: audit 2024-03-07T03:50:20.752559+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.145:0/3612969278' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:50:22.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:21 smithi189 bash[66969]: audit 2024-03-07T03:50:20.752559+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.145:0/3612969278' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:50:22.476 INFO:teuthology.orchestra.run.smithi145.stdout:2074469204345 2024-03-07T03:50:22.576 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2731599200596 got 2731599200600 for osd.2 2024-03-07T03:50:22.577 DEBUG:teuthology.parallel:result is None 2024-03-07T03:50:22.578 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503725103 got 167503725108 for osd.5 2024-03-07T03:50:22.578 DEBUG:teuthology.parallel:result is None 2024-03-07T03:50:22.589 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:50:22 smithi189 bash[59814]: ts=2024-03-07T03:50:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:50:22.757 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:22 smithi145 bash[82329]: cluster 2024-03-07T03:50:21.694183+0000 mgr.y (mgr.34107) 1155 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:22.757 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:22 smithi145 bash[82329]: audit 2024-03-07T03:50:22.478983+0000 mon.a (mon.0) 1975 : audit 0 from='client.? 172.21.15.145:0/1428972864' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:50:22.758 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:22 smithi145 bash[79878]: cluster 2024-03-07T03:50:21.694183+0000 mgr.y (mgr.34107) 1155 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:22.758 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:22 smithi145 bash[79878]: audit 2024-03-07T03:50:22.478983+0000 mon.a (mon.0) 1975 : audit 0 from='client.? 172.21.15.145:0/1428972864' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:50:22.759 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2783138808141 got 2783138808147 for osd.3 2024-03-07T03:50:22.759 DEBUG:teuthology.parallel:result is None 2024-03-07T03:50:22.911 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:50:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:50:22] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.43.0" 2024-03-07T03:50:22.925 INFO:teuthology.orchestra.run.smithi145.stdout:2224793059699 2024-03-07T03:50:23.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:22 smithi189 bash[66969]: cluster 2024-03-07T03:50:21.694183+0000 mgr.y (mgr.34107) 1155 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:23.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:22 smithi189 bash[66969]: audit 2024-03-07T03:50:22.478983+0000 mon.a (mon.0) 1975 : audit 0 from='client.? 172.21.15.145:0/1428972864' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:50:24.087 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:23 smithi145 bash[79878]: audit 2024-03-07T03:50:22.928090+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.145:0/2924044148' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:50:24.088 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:23 smithi145 bash[82329]: audit 2024-03-07T03:50:22.928090+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.145:0/2924044148' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:50:24.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:23 smithi189 bash[66969]: audit 2024-03-07T03:50:22.928090+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.145:0/2924044148' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:50:24.677 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463461 got 201863463467 for osd.6 2024-03-07T03:50:24.678 DEBUG:teuthology.parallel:result is None 2024-03-07T03:50:24.678 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201820 got 236223201824 for osd.7 2024-03-07T03:50:24.678 DEBUG:teuthology.parallel:result is None 2024-03-07T03:50:24.685 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2074469204340 got 2074469204345 for osd.0 2024-03-07T03:50:24.685 DEBUG:teuthology.parallel:result is None 2024-03-07T03:50:24.729 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2224793059693 got 2224793059699 for osd.1 2024-03-07T03:50:24.729 DEBUG:teuthology.parallel:result is None 2024-03-07T03:50:24.729 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-07T03:50:24.729 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph pg dump --format=json 2024-03-07T03:50:25.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:24 smithi145 bash[79878]: cluster 2024-03-07T03:50:23.694789+0000 mgr.y (mgr.34107) 1156 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:25.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:24 smithi145 bash[82329]: cluster 2024-03-07T03:50:23.694789+0000 mgr.y (mgr.34107) 1156 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:25.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:24 smithi189 bash[66969]: cluster 2024-03-07T03:50:23.694789+0000 mgr.y (mgr.34107) 1156 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:27.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:26 smithi145 bash[79878]: cluster 2024-03-07T03:50:25.695524+0000 mgr.y (mgr.34107) 1157 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:27.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:26 smithi145 bash[82329]: cluster 2024-03-07T03:50:25.695524+0000 mgr.y (mgr.34107) 1157 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:27.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:26 smithi189 bash[66969]: cluster 2024-03-07T03:50:25.695524+0000 mgr.y (mgr.34107) 1157 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:29.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:28 smithi145 bash[79878]: cluster 2024-03-07T03:50:27.696480+0000 mgr.y (mgr.34107) 1158 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:29.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:28 smithi145 bash[82329]: cluster 2024-03-07T03:50:27.696480+0000 mgr.y (mgr.34107) 1158 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:29.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:28 smithi189 bash[66969]: cluster 2024-03-07T03:50:27.696480+0000 mgr.y (mgr.34107) 1158 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:29.464 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:50:30.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:30 smithi145 bash[79878]: cluster 2024-03-07T03:50:29.697348+0000 mgr.y (mgr.34107) 1159 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:30.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:30 smithi145 bash[82329]: cluster 2024-03-07T03:50:29.697348+0000 mgr.y (mgr.34107) 1159 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:31.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:30 smithi189 bash[66969]: cluster 2024-03-07T03:50:29.697348+0000 mgr.y (mgr.34107) 1159 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:32.589 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:50:32 smithi189 bash[59814]: ts=2024-03-07T03:50:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:50:32.755 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:50:32.758 INFO:teuthology.orchestra.run.smithi145.stderr:dumped all 2024-03-07T03:50:33.013 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:32 smithi145 bash[79878]: cluster 2024-03-07T03:50:31.698334+0000 mgr.y (mgr.34107) 1160 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:33.014 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:32 smithi145 bash[79878]: audit 2024-03-07T03:50:32.685573+0000 mon.a (mon.0) 1976 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:33.014 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:32 smithi145 bash[82329]: cluster 2024-03-07T03:50:31.698334+0000 mgr.y (mgr.34107) 1160 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:33.014 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:32 smithi145 bash[82329]: audit 2024-03-07T03:50:32.685573+0000 mon.a (mon.0) 1976 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:33.014 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:50:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:50:32] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.43.0" 2024-03-07T03:50:33.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:32 smithi189 bash[66969]: cluster 2024-03-07T03:50:31.698334+0000 mgr.y (mgr.34107) 1160 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:33.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:32 smithi189 bash[66969]: audit 2024-03-07T03:50:32.685573+0000 mon.a (mon.0) 1976 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:33.589 INFO:teuthology.orchestra.run.smithi145.stdout:{"pg_ready":true,"pg_map":{"version":1886,"stamp":"2024-03-07T03:50:31.697667+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":565,"num_read_kb":992,"num_write":300,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":722856,"kb_used_data":100748,"kb_used_omap":331,"kb_used_meta":621684,"kb_avail":749238360,"statfs":{"total":767960285184,"available":767220080640,"internally_reserved":0,"allocated":103165952,"data_stored":65205626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":339384,"internal_metadata":636605000},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004955"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":1522,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.155075+0000","last_change":"2024-03-07T03:22:41.925126+0000","last_active":"2024-03-07T03:49:03.155075+0000","last_peered":"2024-03-07T03:49:03.155075+0000","last_clean":"2024-03-07T03:49:03.155075+0000","last_became_active":"2024-03-07T03:22:41.924495+0000","last_became_peered":"2024-03-07T03:22:41.924495+0000","last_unstale":"2024-03-07T03:49:03.155075+0000","last_undegraded":"2024-03-07T03:49:03.155075+0000","last_fullsized":"2024-03-07T03:49:03.155075+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:47:40.875079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":1654,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.256624+0000","last_change":"2024-03-07T03:21:43.302554+0000","last_active":"2024-03-07T03:49:03.256624+0000","last_peered":"2024-03-07T03:49:03.256624+0000","last_clean":"2024-03-07T03:49:03.256624+0000","last_became_active":"2024-03-07T03:21:43.301987+0000","last_became_peered":"2024-03-07T03:21:43.301987+0000","last_unstale":"2024-03-07T03:49:03.256624+0000","last_undegraded":"2024-03-07T03:49:03.256624+0000","last_fullsized":"2024-03-07T03:49:03.256624+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:57:41.827315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1538,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.256236+0000","last_change":"2024-03-07T03:22:13.255033+0000","last_active":"2024-03-07T03:49:03.256236+0000","last_peered":"2024-03-07T03:49:03.256236+0000","last_clean":"2024-03-07T03:49:03.256236+0000","last_became_active":"2024-03-07T03:22:13.254227+0000","last_became_peered":"2024-03-07T03:22:13.254227+0000","last_unstale":"2024-03-07T03:49:03.256236+0000","last_undegraded":"2024-03-07T03:49:03.256236+0000","last_fullsized":"2024-03-07T03:49:03.256236+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:49:30.698453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":1530,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.257190+0000","last_change":"2024-03-07T03:22:13.280263+0000","last_active":"2024-03-07T03:49:03.257190+0000","last_peered":"2024-03-07T03:49:03.257190+0000","last_clean":"2024-03-07T03:49:03.257190+0000","last_became_active":"2024-03-07T03:22:13.279394+0000","last_became_peered":"2024-03-07T03:22:13.279394+0000","last_unstale":"2024-03-07T03:49:03.257190+0000","last_undegraded":"2024-03-07T03:49:03.257190+0000","last_fullsized":"2024-03-07T03:49:03.257190+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:31:37.602107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1537,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.161211+0000","last_change":"2024-03-07T03:22:41.953431+0000","last_active":"2024-03-07T03:49:03.161211+0000","last_peered":"2024-03-07T03:49:03.161211+0000","last_clean":"2024-03-07T03:49:03.161211+0000","last_became_active":"2024-03-07T03:22:41.946313+0000","last_became_peered":"2024-03-07T03:22:41.946313+0000","last_unstale":"2024-03-07T03:49:03.161211+0000","last_undegraded":"2024-03-07T03:49:03.161211+0000","last_fullsized":"2024-03-07T03:49:03.161211+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:47:23.279612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":1066,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.144779+0000","last_change":"2024-03-07T03:22:41.935909+0000","last_active":"2024-03-07T03:48:58.144779+0000","last_peered":"2024-03-07T03:48:58.144779+0000","last_clean":"2024-03-07T03:48:58.144779+0000","last_became_active":"2024-03-07T03:22:41.935561+0000","last_became_peered":"2024-03-07T03:22:41.935561+0000","last_unstale":"2024-03-07T03:48:58.144779+0000","last_undegraded":"2024-03-07T03:48:58.144779+0000","last_fullsized":"2024-03-07T03:48:58.144779+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:09:48.621152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1537,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.157195+0000","last_change":"2024-03-07T03:22:41.931999+0000","last_active":"2024-03-07T03:49:03.157195+0000","last_peered":"2024-03-07T03:49:03.157195+0000","last_clean":"2024-03-07T03:49:03.157195+0000","last_became_active":"2024-03-07T03:22:41.931590+0000","last_became_peered":"2024-03-07T03:22:41.931590+0000","last_unstale":"2024-03-07T03:49:03.157195+0000","last_undegraded":"2024-03-07T03:49:03.157195+0000","last_fullsized":"2024-03-07T03:49:03.157195+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:46:30.686424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":1081,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.144521+0000","last_change":"2024-03-07T03:22:41.935688+0000","last_active":"2024-03-07T03:48:58.144521+0000","last_peered":"2024-03-07T03:48:58.144521+0000","last_clean":"2024-03-07T03:48:58.144521+0000","last_became_active":"2024-03-07T03:22:41.935333+0000","last_became_peered":"2024-03-07T03:22:41.935333+0000","last_unstale":"2024-03-07T03:48:58.144521+0000","last_undegraded":"2024-03-07T03:48:58.144521+0000","last_fullsized":"2024-03-07T03:48:58.144521+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:05:22.450810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1512,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.257799+0000","last_change":"2024-03-07T03:22:13.266027+0000","last_active":"2024-03-07T03:49:03.257799+0000","last_peered":"2024-03-07T03:49:03.257799+0000","last_clean":"2024-03-07T03:49:03.257799+0000","last_became_active":"2024-03-07T03:22:13.265741+0000","last_became_peered":"2024-03-07T03:22:13.265741+0000","last_unstale":"2024-03-07T03:49:03.257799+0000","last_undegraded":"2024-03-07T03:49:03.257799+0000","last_fullsized":"2024-03-07T03:49:03.257799+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:54:16.621812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":1645,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253149+0000","last_change":"2024-03-07T03:22:41.922062+0000","last_active":"2024-03-07T03:49:03.253149+0000","last_peered":"2024-03-07T03:49:03.253149+0000","last_clean":"2024-03-07T03:49:03.253149+0000","last_became_active":"2024-03-07T03:22:41.921629+0000","last_became_peered":"2024-03-07T03:22:41.921629+0000","last_unstale":"2024-03-07T03:49:03.253149+0000","last_undegraded":"2024-03-07T03:49:03.253149+0000","last_fullsized":"2024-03-07T03:49:03.253149+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:00:57.248696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1510,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.159076+0000","last_change":"2024-03-07T03:22:41.957347+0000","last_active":"2024-03-07T03:49:03.159076+0000","last_peered":"2024-03-07T03:49:03.159076+0000","last_clean":"2024-03-07T03:49:03.159076+0000","last_became_active":"2024-03-07T03:22:41.952518+0000","last_became_peered":"2024-03-07T03:22:41.952518+0000","last_unstale":"2024-03-07T03:49:03.159076+0000","last_undegraded":"2024-03-07T03:49:03.159076+0000","last_fullsized":"2024-03-07T03:49:03.159076+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:15:57.274707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":1656,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.252207+0000","last_change":"2024-03-07T03:22:41.921898+0000","last_active":"2024-03-07T03:49:03.252207+0000","last_peered":"2024-03-07T03:49:03.252207+0000","last_clean":"2024-03-07T03:49:03.252207+0000","last_became_active":"2024-03-07T03:22:41.921297+0000","last_became_peered":"2024-03-07T03:22:41.921297+0000","last_unstale":"2024-03-07T03:49:03.252207+0000","last_undegraded":"2024-03-07T03:49:03.252207+0000","last_fullsized":"2024-03-07T03:49:03.252207+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:09:06.393622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1075,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.151945+0000","last_change":"2024-03-07T03:21:43.304121+0000","last_active":"2024-03-07T03:49:03.151945+0000","last_peered":"2024-03-07T03:49:03.151945+0000","last_clean":"2024-03-07T03:49:03.151945+0000","last_became_active":"2024-03-07T03:21:43.302961+0000","last_became_peered":"2024-03-07T03:21:43.302961+0000","last_unstale":"2024-03-07T03:49:03.151945+0000","last_undegraded":"2024-03-07T03:49:03.151945+0000","last_fullsized":"2024-03-07T03:49:03.151945+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:13:38.289491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1527,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253421+0000","last_change":"2024-03-07T03:22:13.301668+0000","last_active":"2024-03-07T03:49:03.253421+0000","last_peered":"2024-03-07T03:49:03.253421+0000","last_clean":"2024-03-07T03:49:03.253421+0000","last_became_active":"2024-03-07T03:22:13.298895+0000","last_became_peered":"2024-03-07T03:22:13.298895+0000","last_unstale":"2024-03-07T03:49:03.253421+0000","last_undegraded":"2024-03-07T03:49:03.253421+0000","last_fullsized":"2024-03-07T03:49:03.253421+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:57:04.778318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1618,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.277039+0000","last_change":"2024-03-07T03:22:41.908219+0000","last_active":"2024-03-07T03:49:03.277039+0000","last_peered":"2024-03-07T03:49:03.277039+0000","last_clean":"2024-03-07T03:49:03.277039+0000","last_became_active":"2024-03-07T03:22:41.907902+0000","last_became_peered":"2024-03-07T03:22:41.907902+0000","last_unstale":"2024-03-07T03:49:03.277039+0000","last_undegraded":"2024-03-07T03:49:03.277039+0000","last_fullsized":"2024-03-07T03:49:03.277039+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:08:33.376548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":1494,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.256668+0000","last_change":"2024-03-07T03:22:13.292154+0000","last_active":"2024-03-07T03:49:03.256668+0000","last_peered":"2024-03-07T03:49:03.256668+0000","last_clean":"2024-03-07T03:49:03.256668+0000","last_became_active":"2024-03-07T03:22:13.291446+0000","last_became_peered":"2024-03-07T03:22:13.291446+0000","last_unstale":"2024-03-07T03:49:03.256668+0000","last_undegraded":"2024-03-07T03:49:03.256668+0000","last_fullsized":"2024-03-07T03:49:03.256668+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:16:03.766353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1635,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253066+0000","last_change":"2024-03-07T03:19:03.742695+0000","last_active":"2024-03-07T03:49:03.253066+0000","last_peered":"2024-03-07T03:49:03.253066+0000","last_clean":"2024-03-07T03:49:03.253066+0000","last_became_active":"2024-03-07T03:19:03.741911+0000","last_became_peered":"2024-03-07T03:19:03.741911+0000","last_unstale":"2024-03-07T03:49:03.253066+0000","last_undegraded":"2024-03-07T03:49:03.253066+0000","last_fullsized":"2024-03-07T03:49:03.253066+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:21:55.513877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1618,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.271691+0000","last_change":"2024-03-07T03:21:43.299149+0000","last_active":"2024-03-07T03:49:03.271691+0000","last_peered":"2024-03-07T03:49:03.271691+0000","last_clean":"2024-03-07T03:49:03.271691+0000","last_became_active":"2024-03-07T03:21:43.297777+0000","last_became_peered":"2024-03-07T03:21:43.297777+0000","last_unstale":"2024-03-07T03:49:03.271691+0000","last_undegraded":"2024-03-07T03:49:03.271691+0000","last_fullsized":"2024-03-07T03:49:03.271691+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:51:07.610242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1089,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.885844+0000","last_change":"2024-03-07T03:22:41.903154+0000","last_active":"2024-03-07T03:48:58.885844+0000","last_peered":"2024-03-07T03:48:58.885844+0000","last_clean":"2024-03-07T03:48:58.885844+0000","last_became_active":"2024-03-07T03:22:41.902822+0000","last_became_peered":"2024-03-07T03:22:41.902822+0000","last_unstale":"2024-03-07T03:48:58.885844+0000","last_undegraded":"2024-03-07T03:48:58.885844+0000","last_fullsized":"2024-03-07T03:48:58.885844+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:44:07.344147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1652,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.252498+0000","last_change":"2024-03-07T03:21:43.305697+0000","last_active":"2024-03-07T03:49:03.252498+0000","last_peered":"2024-03-07T03:49:03.252498+0000","last_clean":"2024-03-07T03:49:03.252498+0000","last_became_active":"2024-03-07T03:21:43.304986+0000","last_became_peered":"2024-03-07T03:21:43.304986+0000","last_unstale":"2024-03-07T03:49:03.252498+0000","last_undegraded":"2024-03-07T03:49:03.252498+0000","last_fullsized":"2024-03-07T03:49:03.252498+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:25:48.065258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1652,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.251610+0000","last_change":"2024-03-07T03:22:41.924664+0000","last_active":"2024-03-07T03:49:03.251610+0000","last_peered":"2024-03-07T03:49:03.251610+0000","last_clean":"2024-03-07T03:49:03.251610+0000","last_became_active":"2024-03-07T03:22:41.923356+0000","last_became_peered":"2024-03-07T03:22:41.923356+0000","last_unstale":"2024-03-07T03:49:03.251610+0000","last_undegraded":"2024-03-07T03:49:03.251610+0000","last_fullsized":"2024-03-07T03:49:03.251610+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:43:47.698590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1090,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.883631+0000","last_change":"2024-03-07T03:19:38.525605+0000","last_active":"2024-03-07T03:48:58.883631+0000","last_peered":"2024-03-07T03:48:58.883631+0000","last_clean":"2024-03-07T03:48:58.883631+0000","last_became_active":"2024-03-07T03:19:38.524575+0000","last_became_peered":"2024-03-07T03:19:38.524575+0000","last_unstale":"2024-03-07T03:48:58.883631+0000","last_undegraded":"2024-03-07T03:48:58.883631+0000","last_fullsized":"2024-03-07T03:48:58.883631+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:21:46.949046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1069,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.964954+0000","last_change":"2024-03-07T03:22:41.924297+0000","last_active":"2024-03-07T03:48:58.964954+0000","last_peered":"2024-03-07T03:48:58.964954+0000","last_clean":"2024-03-07T03:48:58.964954+0000","last_became_active":"2024-03-07T03:22:41.923679+0000","last_became_peered":"2024-03-07T03:22:41.923679+0000","last_unstale":"2024-03-07T03:48:58.964954+0000","last_undegraded":"2024-03-07T03:48:58.964954+0000","last_fullsized":"2024-03-07T03:48:58.964954+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:22:51.253808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1512,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.251735+0000","last_change":"2024-03-07T03:22:13.289010+0000","last_active":"2024-03-07T03:49:03.251735+0000","last_peered":"2024-03-07T03:49:03.251735+0000","last_clean":"2024-03-07T03:49:03.251735+0000","last_became_active":"2024-03-07T03:22:13.288665+0000","last_became_peered":"2024-03-07T03:22:13.288665+0000","last_unstale":"2024-03-07T03:49:03.251735+0000","last_undegraded":"2024-03-07T03:49:03.251735+0000","last_fullsized":"2024-03-07T03:49:03.251735+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:36:57.801158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"284'161","reported_seq":1440,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.156109+0000","last_change":"2024-03-07T03:19:03.750010+0000","last_active":"2024-03-07T03:49:03.156109+0000","last_peered":"2024-03-07T03:49:03.156109+0000","last_clean":"2024-03-07T03:49:03.156109+0000","last_became_active":"2024-03-07T03:19:03.749484+0000","last_became_peered":"2024-03-07T03:19:03.749484+0000","last_unstale":"2024-03-07T03:49:03.156109+0000","last_undegraded":"2024-03-07T03:49:03.156109+0000","last_fullsized":"2024-03-07T03:49:03.156109+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:01:00.212577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:01:00.212577+0000","last_clean_scrub_stamp":"2024-03-07T03:01:00.212577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:00:56.725739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1071,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.255232+0000","last_change":"2024-03-07T03:21:43.269933+0000","last_active":"2024-03-07T03:49:03.255232+0000","last_peered":"2024-03-07T03:49:03.255232+0000","last_clean":"2024-03-07T03:49:03.255232+0000","last_became_active":"2024-03-07T03:21:43.269289+0000","last_became_peered":"2024-03-07T03:21:43.269289+0000","last_unstale":"2024-03-07T03:49:03.255232+0000","last_undegraded":"2024-03-07T03:49:03.255232+0000","last_fullsized":"2024-03-07T03:49:03.255232+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:09:13.977042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1629,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.287624+0000","last_change":"2024-03-07T03:19:38.544337+0000","last_active":"2024-03-07T03:49:03.287624+0000","last_peered":"2024-03-07T03:49:03.287624+0000","last_clean":"2024-03-07T03:49:03.287624+0000","last_became_active":"2024-03-07T03:19:38.541610+0000","last_became_peered":"2024-03-07T03:19:38.541610+0000","last_unstale":"2024-03-07T03:49:03.287624+0000","last_undegraded":"2024-03-07T03:49:03.287624+0000","last_fullsized":"2024-03-07T03:49:03.287624+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:06:15.042560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1620,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.264891+0000","last_change":"2024-03-07T03:21:43.281389+0000","last_active":"2024-03-07T03:49:03.264891+0000","last_peered":"2024-03-07T03:49:03.264891+0000","last_clean":"2024-03-07T03:49:03.264891+0000","last_became_active":"2024-03-07T03:21:43.278292+0000","last_became_peered":"2024-03-07T03:21:43.278292+0000","last_unstale":"2024-03-07T03:49:03.264891+0000","last_undegraded":"2024-03-07T03:49:03.264891+0000","last_fullsized":"2024-03-07T03:49:03.264891+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:13:19.332308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1545,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.251961+0000","last_change":"2024-03-07T03:22:13.239633+0000","last_active":"2024-03-07T03:49:03.251961+0000","last_peered":"2024-03-07T03:49:03.251961+0000","last_clean":"2024-03-07T03:49:03.251961+0000","last_became_active":"2024-03-07T03:22:13.239322+0000","last_became_peered":"2024-03-07T03:22:13.239322+0000","last_unstale":"2024-03-07T03:49:03.251961+0000","last_undegraded":"2024-03-07T03:49:03.251961+0000","last_fullsized":"2024-03-07T03:49:03.251961+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:40:53.579559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1075,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.144133+0000","last_change":"2024-03-07T03:22:13.276342+0000","last_active":"2024-03-07T03:48:58.144133+0000","last_peered":"2024-03-07T03:48:58.144133+0000","last_clean":"2024-03-07T03:48:58.144133+0000","last_became_active":"2024-03-07T03:22:13.276074+0000","last_became_peered":"2024-03-07T03:22:13.276074+0000","last_unstale":"2024-03-07T03:48:58.144133+0000","last_undegraded":"2024-03-07T03:48:58.144133+0000","last_fullsized":"2024-03-07T03:48:58.144133+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:20:30.055957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1523,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.158042+0000","last_change":"2024-03-07T03:22:41.960515+0000","last_active":"2024-03-07T03:49:03.158042+0000","last_peered":"2024-03-07T03:49:03.158042+0000","last_clean":"2024-03-07T03:49:03.158042+0000","last_became_active":"2024-03-07T03:22:41.953664+0000","last_became_peered":"2024-03-07T03:22:41.953664+0000","last_unstale":"2024-03-07T03:49:03.158042+0000","last_undegraded":"2024-03-07T03:49:03.158042+0000","last_fullsized":"2024-03-07T03:49:03.158042+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:55:02.230660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1071,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.250765+0000","last_change":"2024-03-07T03:19:38.522178+0000","last_active":"2024-03-07T03:49:03.250765+0000","last_peered":"2024-03-07T03:49:03.250765+0000","last_clean":"2024-03-07T03:49:03.250765+0000","last_became_active":"2024-03-07T03:19:38.521276+0000","last_became_peered":"2024-03-07T03:19:38.521276+0000","last_unstale":"2024-03-07T03:49:03.250765+0000","last_undegraded":"2024-03-07T03:49:03.250765+0000","last_fullsized":"2024-03-07T03:49:03.250765+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:24:42.660567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1618,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.268690+0000","last_change":"2024-03-07T03:22:41.885534+0000","last_active":"2024-03-07T03:49:03.268690+0000","last_peered":"2024-03-07T03:49:03.268690+0000","last_clean":"2024-03-07T03:49:03.268690+0000","last_became_active":"2024-03-07T03:22:41.885340+0000","last_became_peered":"2024-03-07T03:22:41.885340+0000","last_unstale":"2024-03-07T03:49:03.268690+0000","last_undegraded":"2024-03-07T03:49:03.268690+0000","last_fullsized":"2024-03-07T03:49:03.268690+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:35:17.363850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1080,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.144290+0000","last_change":"2024-03-07T03:22:13.298793+0000","last_active":"2024-03-07T03:48:58.144290+0000","last_peered":"2024-03-07T03:48:58.144290+0000","last_clean":"2024-03-07T03:48:58.144290+0000","last_became_active":"2024-03-07T03:22:13.298480+0000","last_became_peered":"2024-03-07T03:22:13.298480+0000","last_unstale":"2024-03-07T03:48:58.144290+0000","last_undegraded":"2024-03-07T03:48:58.144290+0000","last_fullsized":"2024-03-07T03:48:58.144290+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:08:08.449442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1635,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.284706+0000","last_change":"2024-03-07T03:19:38.532550+0000","last_active":"2024-03-07T03:49:03.284706+0000","last_peered":"2024-03-07T03:49:03.284706+0000","last_clean":"2024-03-07T03:49:03.284706+0000","last_became_active":"2024-03-07T03:19:38.531243+0000","last_became_peered":"2024-03-07T03:19:38.531243+0000","last_unstale":"2024-03-07T03:49:03.284706+0000","last_undegraded":"2024-03-07T03:49:03.284706+0000","last_fullsized":"2024-03-07T03:49:03.284706+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:21:28.043587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1626,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.287462+0000","last_change":"2024-03-07T03:22:41.909258+0000","last_active":"2024-03-07T03:49:03.287462+0000","last_peered":"2024-03-07T03:49:03.287462+0000","last_clean":"2024-03-07T03:49:03.287462+0000","last_became_active":"2024-03-07T03:22:41.908514+0000","last_became_peered":"2024-03-07T03:22:41.908514+0000","last_unstale":"2024-03-07T03:49:03.287462+0000","last_undegraded":"2024-03-07T03:49:03.287462+0000","last_fullsized":"2024-03-07T03:49:03.287462+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:38:57.312991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1527,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253012+0000","last_change":"2024-03-07T03:22:13.304093+0000","last_active":"2024-03-07T03:49:03.253012+0000","last_peered":"2024-03-07T03:49:03.253012+0000","last_clean":"2024-03-07T03:49:03.253012+0000","last_became_active":"2024-03-07T03:22:13.301755+0000","last_became_peered":"2024-03-07T03:22:13.301755+0000","last_unstale":"2024-03-07T03:49:03.253012+0000","last_undegraded":"2024-03-07T03:49:03.253012+0000","last_fullsized":"2024-03-07T03:49:03.253012+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:05:55.915533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1514,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.167462+0000","last_change":"2024-03-07T03:22:41.926183+0000","last_active":"2024-03-07T03:49:03.167462+0000","last_peered":"2024-03-07T03:49:03.167462+0000","last_clean":"2024-03-07T03:49:03.167462+0000","last_became_active":"2024-03-07T03:22:41.925842+0000","last_became_peered":"2024-03-07T03:22:41.925842+0000","last_unstale":"2024-03-07T03:49:03.167462+0000","last_undegraded":"2024-03-07T03:49:03.167462+0000","last_fullsized":"2024-03-07T03:49:03.167462+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:55:23.626786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1525,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.153461+0000","last_change":"2024-03-07T03:22:41.956505+0000","last_active":"2024-03-07T03:49:03.153461+0000","last_peered":"2024-03-07T03:49:03.153461+0000","last_clean":"2024-03-07T03:49:03.153461+0000","last_became_active":"2024-03-07T03:22:41.943938+0000","last_became_peered":"2024-03-07T03:22:41.943938+0000","last_unstale":"2024-03-07T03:49:03.153461+0000","last_undegraded":"2024-03-07T03:49:03.153461+0000","last_fullsized":"2024-03-07T03:49:03.153461+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:37:04.758716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1517,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.165122+0000","last_change":"2024-03-07T03:22:41.914323+0000","last_active":"2024-03-07T03:49:03.165122+0000","last_peered":"2024-03-07T03:49:03.165122+0000","last_clean":"2024-03-07T03:49:03.165122+0000","last_became_active":"2024-03-07T03:22:41.913344+0000","last_became_peered":"2024-03-07T03:22:41.913344+0000","last_unstale":"2024-03-07T03:49:03.165122+0000","last_undegraded":"2024-03-07T03:49:03.165122+0000","last_fullsized":"2024-03-07T03:49:03.165122+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:30:59.510000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1085,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.149079+0000","last_change":"2024-03-07T03:22:41.922381+0000","last_active":"2024-03-07T03:49:03.149079+0000","last_peered":"2024-03-07T03:49:03.149079+0000","last_clean":"2024-03-07T03:49:03.149079+0000","last_became_active":"2024-03-07T03:22:41.921959+0000","last_became_peered":"2024-03-07T03:22:41.921959+0000","last_unstale":"2024-03-07T03:49:03.149079+0000","last_undegraded":"2024-03-07T03:49:03.149079+0000","last_fullsized":"2024-03-07T03:49:03.149079+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:32:00.451279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1520,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.158239+0000","last_change":"2024-03-07T03:22:41.957256+0000","last_active":"2024-03-07T03:49:03.158239+0000","last_peered":"2024-03-07T03:49:03.158239+0000","last_clean":"2024-03-07T03:49:03.158239+0000","last_became_active":"2024-03-07T03:22:41.947708+0000","last_became_peered":"2024-03-07T03:22:41.947708+0000","last_unstale":"2024-03-07T03:49:03.158239+0000","last_undegraded":"2024-03-07T03:49:03.158239+0000","last_fullsized":"2024-03-07T03:49:03.158239+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:49:19.355156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":1538,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253954+0000","last_change":"2024-03-07T03:22:41.931128+0000","last_active":"2024-03-07T03:49:03.253954+0000","last_peered":"2024-03-07T03:49:03.253954+0000","last_clean":"2024-03-07T03:49:03.253954+0000","last_became_active":"2024-03-07T03:22:41.930808+0000","last_became_peered":"2024-03-07T03:22:41.930808+0000","last_unstale":"2024-03-07T03:49:03.253954+0000","last_undegraded":"2024-03-07T03:49:03.253954+0000","last_fullsized":"2024-03-07T03:49:03.253954+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:01:52.246153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1664,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.252745+0000","last_change":"2024-03-07T03:22:41.923532+0000","last_active":"2024-03-07T03:49:03.252745+0000","last_peered":"2024-03-07T03:49:03.252745+0000","last_clean":"2024-03-07T03:49:03.252745+0000","last_became_active":"2024-03-07T03:22:41.923218+0000","last_became_peered":"2024-03-07T03:22:41.923218+0000","last_unstale":"2024-03-07T03:49:03.252745+0000","last_undegraded":"2024-03-07T03:49:03.252745+0000","last_fullsized":"2024-03-07T03:49:03.252745+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:13:19.255516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1064,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.252480+0000","last_change":"2024-03-07T03:22:13.302012+0000","last_active":"2024-03-07T03:49:03.252480+0000","last_peered":"2024-03-07T03:49:03.252480+0000","last_clean":"2024-03-07T03:49:03.252480+0000","last_became_active":"2024-03-07T03:22:13.301560+0000","last_became_peered":"2024-03-07T03:22:13.301560+0000","last_unstale":"2024-03-07T03:49:03.252480+0000","last_undegraded":"2024-03-07T03:49:03.252480+0000","last_fullsized":"2024-03-07T03:49:03.252480+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:43:14.805124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1069,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.148656+0000","last_change":"2024-03-07T03:19:38.535906+0000","last_active":"2024-03-07T03:49:03.148656+0000","last_peered":"2024-03-07T03:49:03.148656+0000","last_clean":"2024-03-07T03:49:03.148656+0000","last_became_active":"2024-03-07T03:19:38.535036+0000","last_became_peered":"2024-03-07T03:19:38.535036+0000","last_unstale":"2024-03-07T03:49:03.148656+0000","last_undegraded":"2024-03-07T03:49:03.148656+0000","last_fullsized":"2024-03-07T03:49:03.148656+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:21:05.129278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":1055,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.162376+0000","last_change":"2024-03-07T03:08:49.421752+0000","last_active":"2024-03-07T03:49:03.162376+0000","last_peered":"2024-03-07T03:49:03.162376+0000","last_clean":"2024-03-07T03:49:03.162376+0000","last_became_active":"2024-03-07T03:08:49.421165+0000","last_became_peered":"2024-03-07T03:08:49.421165+0000","last_unstale":"2024-03-07T03:49:03.162376+0000","last_undegraded":"2024-03-07T03:49:03.162376+0000","last_fullsized":"2024-03-07T03:49:03.162376+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:57:59.044174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1539,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.159223+0000","last_change":"2024-03-07T03:22:41.931071+0000","last_active":"2024-03-07T03:49:03.159223+0000","last_peered":"2024-03-07T03:49:03.159223+0000","last_clean":"2024-03-07T03:49:03.159223+0000","last_became_active":"2024-03-07T03:22:41.930732+0000","last_became_peered":"2024-03-07T03:22:41.930732+0000","last_unstale":"2024-03-07T03:49:03.159223+0000","last_undegraded":"2024-03-07T03:49:03.159223+0000","last_fullsized":"2024-03-07T03:49:03.159223+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:44:46.520072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":1087,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.163516+0000","last_change":"2024-03-07T03:22:41.904040+0000","last_active":"2024-03-07T03:49:03.163516+0000","last_peered":"2024-03-07T03:49:03.163516+0000","last_clean":"2024-03-07T03:49:03.163516+0000","last_became_active":"2024-03-07T03:22:41.903638+0000","last_became_peered":"2024-03-07T03:22:41.903638+0000","last_unstale":"2024-03-07T03:49:03.163516+0000","last_undegraded":"2024-03-07T03:49:03.163516+0000","last_fullsized":"2024-03-07T03:49:03.163516+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:24:34.364981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1049,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.250205+0000","last_change":"2024-03-07T03:08:50.110958+0000","last_active":"2024-03-07T03:49:03.250205+0000","last_peered":"2024-03-07T03:49:03.250205+0000","last_clean":"2024-03-07T03:49:03.250205+0000","last_became_active":"2024-03-07T03:08:50.110066+0000","last_became_peered":"2024-03-07T03:08:50.110066+0000","last_unstale":"2024-03-07T03:49:03.250205+0000","last_undegraded":"2024-03-07T03:49:03.250205+0000","last_fullsized":"2024-03-07T03:49:03.250205+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:57:28.837409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":1085,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.166054+0000","last_change":"2024-03-07T03:22:41.906951+0000","last_active":"2024-03-07T03:49:03.166054+0000","last_peered":"2024-03-07T03:49:03.166054+0000","last_clean":"2024-03-07T03:49:03.166054+0000","last_became_active":"2024-03-07T03:22:41.906609+0000","last_became_peered":"2024-03-07T03:22:41.906609+0000","last_unstale":"2024-03-07T03:49:03.166054+0000","last_undegraded":"2024-03-07T03:49:03.166054+0000","last_fullsized":"2024-03-07T03:49:03.166054+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:13:25.180063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1522,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.255641+0000","last_change":"2024-03-07T03:22:13.298545+0000","last_active":"2024-03-07T03:49:03.255641+0000","last_peered":"2024-03-07T03:49:03.255641+0000","last_clean":"2024-03-07T03:49:03.255641+0000","last_became_active":"2024-03-07T03:22:13.298225+0000","last_became_peered":"2024-03-07T03:22:13.298225+0000","last_unstale":"2024-03-07T03:49:03.255641+0000","last_undegraded":"2024-03-07T03:49:03.255641+0000","last_fullsized":"2024-03-07T03:49:03.255641+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:10:15.575412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":1084,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.963172+0000","last_change":"2024-03-07T03:19:03.727952+0000","last_active":"2024-03-07T03:48:58.963172+0000","last_peered":"2024-03-07T03:48:58.963172+0000","last_clean":"2024-03-07T03:48:58.963172+0000","last_became_active":"2024-03-07T03:19:03.727379+0000","last_became_peered":"2024-03-07T03:19:03.727379+0000","last_unstale":"2024-03-07T03:48:58.963172+0000","last_undegraded":"2024-03-07T03:48:58.963172+0000","last_fullsized":"2024-03-07T03:48:58.963172+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:02:49.830245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1641,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.251895+0000","last_change":"2024-03-07T03:22:13.285040+0000","last_active":"2024-03-07T03:49:03.251895+0000","last_peered":"2024-03-07T03:49:03.251895+0000","last_clean":"2024-03-07T03:49:03.251895+0000","last_became_active":"2024-03-07T03:22:13.284262+0000","last_became_peered":"2024-03-07T03:22:13.284262+0000","last_unstale":"2024-03-07T03:49:03.251895+0000","last_undegraded":"2024-03-07T03:49:03.251895+0000","last_fullsized":"2024-03-07T03:49:03.251895+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:20:44.532634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1067,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.884958+0000","last_change":"2024-03-07T03:22:41.932835+0000","last_active":"2024-03-07T03:48:58.884958+0000","last_peered":"2024-03-07T03:48:58.884958+0000","last_clean":"2024-03-07T03:48:58.884958+0000","last_became_active":"2024-03-07T03:22:41.932544+0000","last_became_peered":"2024-03-07T03:22:41.932544+0000","last_unstale":"2024-03-07T03:48:58.884958+0000","last_undegraded":"2024-03-07T03:48:58.884958+0000","last_fullsized":"2024-03-07T03:48:58.884958+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:57:31.879866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1062,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.252741+0000","last_change":"2024-03-07T03:22:13.307826+0000","last_active":"2024-03-07T03:49:03.252741+0000","last_peered":"2024-03-07T03:49:03.252741+0000","last_clean":"2024-03-07T03:49:03.252741+0000","last_became_active":"2024-03-07T03:22:13.306153+0000","last_became_peered":"2024-03-07T03:22:13.306153+0000","last_unstale":"2024-03-07T03:49:03.252741+0000","last_undegraded":"2024-03-07T03:49:03.252741+0000","last_fullsized":"2024-03-07T03:49:03.252741+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:28:23.430883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1640,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.254200+0000","last_change":"2024-03-07T03:22:13.284663+0000","last_active":"2024-03-07T03:49:03.254200+0000","last_peered":"2024-03-07T03:49:03.254200+0000","last_clean":"2024-03-07T03:49:03.254200+0000","last_became_active":"2024-03-07T03:22:13.284296+0000","last_became_peered":"2024-03-07T03:22:13.284296+0000","last_unstale":"2024-03-07T03:49:03.254200+0000","last_undegraded":"2024-03-07T03:49:03.254200+0000","last_fullsized":"2024-03-07T03:49:03.254200+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:27:27.139954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1523,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.261804+0000","last_change":"2024-03-07T03:22:13.300789+0000","last_active":"2024-03-07T03:49:03.261804+0000","last_peered":"2024-03-07T03:49:03.261804+0000","last_clean":"2024-03-07T03:49:03.261804+0000","last_became_active":"2024-03-07T03:22:13.295435+0000","last_became_peered":"2024-03-07T03:22:13.295435+0000","last_unstale":"2024-03-07T03:49:03.261804+0000","last_undegraded":"2024-03-07T03:49:03.261804+0000","last_fullsized":"2024-03-07T03:49:03.261804+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:24:33.504833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":1086,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.885558+0000","last_change":"2024-03-07T03:22:41.932597+0000","last_active":"2024-03-07T03:48:58.885558+0000","last_peered":"2024-03-07T03:48:58.885558+0000","last_clean":"2024-03-07T03:48:58.885558+0000","last_became_active":"2024-03-07T03:22:41.931421+0000","last_became_peered":"2024-03-07T03:22:41.931421+0000","last_unstale":"2024-03-07T03:48:58.885558+0000","last_undegraded":"2024-03-07T03:48:58.885558+0000","last_fullsized":"2024-03-07T03:48:58.885558+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:43:47.536816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":8,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1522,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.157666+0000","last_change":"2024-03-07T03:22:41.929992+0000","last_active":"2024-03-07T03:49:03.157666+0000","last_peered":"2024-03-07T03:49:03.157666+0000","last_clean":"2024-03-07T03:49:03.157666+0000","last_became_active":"2024-03-07T03:22:41.928923+0000","last_became_peered":"2024-03-07T03:22:41.928923+0000","last_unstale":"2024-03-07T03:49:03.157666+0000","last_undegraded":"2024-03-07T03:49:03.157666+0000","last_fullsized":"2024-03-07T03:49:03.157666+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:03:09.890598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"338'23","reported_seq":1511,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.158385+0000","last_change":"2024-03-07T03:22:41.958079+0000","last_active":"2024-03-07T03:49:03.158385+0000","last_peered":"2024-03-07T03:49:03.158385+0000","last_clean":"2024-03-07T03:49:03.158385+0000","last_became_active":"2024-03-07T03:22:41.949612+0000","last_became_peered":"2024-03-07T03:22:41.949612+0000","last_unstale":"2024-03-07T03:49:03.158385+0000","last_undegraded":"2024-03-07T03:49:03.158385+0000","last_fullsized":"2024-03-07T03:49:03.158385+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:05:27.967329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1617,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.268009+0000","last_change":"2024-03-07T03:21:43.279937+0000","last_active":"2024-03-07T03:49:03.268009+0000","last_peered":"2024-03-07T03:49:03.268009+0000","last_clean":"2024-03-07T03:49:03.268009+0000","last_became_active":"2024-03-07T03:21:43.279154+0000","last_became_peered":"2024-03-07T03:21:43.279154+0000","last_unstale":"2024-03-07T03:49:03.268009+0000","last_undegraded":"2024-03-07T03:49:03.268009+0000","last_fullsized":"2024-03-07T03:49:03.268009+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:54:02.755057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":1088,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.162781+0000","last_change":"2024-03-07T03:22:41.904810+0000","last_active":"2024-03-07T03:49:03.162781+0000","last_peered":"2024-03-07T03:49:03.162781+0000","last_clean":"2024-03-07T03:49:03.162781+0000","last_became_active":"2024-03-07T03:22:41.903580+0000","last_became_peered":"2024-03-07T03:22:41.903580+0000","last_unstale":"2024-03-07T03:49:03.162781+0000","last_undegraded":"2024-03-07T03:49:03.162781+0000","last_fullsized":"2024-03-07T03:49:03.162781+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:24:19.771759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1509,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.256753+0000","last_change":"2024-03-07T03:22:13.291868+0000","last_active":"2024-03-07T03:49:03.256753+0000","last_peered":"2024-03-07T03:49:03.256753+0000","last_clean":"2024-03-07T03:49:03.256753+0000","last_became_active":"2024-03-07T03:22:13.291433+0000","last_became_peered":"2024-03-07T03:22:13.291433+0000","last_unstale":"2024-03-07T03:49:03.256753+0000","last_undegraded":"2024-03-07T03:49:03.256753+0000","last_fullsized":"2024-03-07T03:49:03.256753+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:37:17.893023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1063,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.252175+0000","last_change":"2024-03-07T03:19:38.536795+0000","last_active":"2024-03-07T03:49:03.252175+0000","last_peered":"2024-03-07T03:49:03.252175+0000","last_clean":"2024-03-07T03:49:03.252175+0000","last_became_active":"2024-03-07T03:19:38.535022+0000","last_became_peered":"2024-03-07T03:19:38.535022+0000","last_unstale":"2024-03-07T03:49:03.252175+0000","last_undegraded":"2024-03-07T03:49:03.252175+0000","last_fullsized":"2024-03-07T03:49:03.252175+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:35:50.615645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1524,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253263+0000","last_change":"2024-03-07T03:22:13.280490+0000","last_active":"2024-03-07T03:49:03.253263+0000","last_peered":"2024-03-07T03:49:03.253263+0000","last_clean":"2024-03-07T03:49:03.253263+0000","last_became_active":"2024-03-07T03:22:13.280062+0000","last_became_peered":"2024-03-07T03:22:13.280062+0000","last_unstale":"2024-03-07T03:49:03.253263+0000","last_undegraded":"2024-03-07T03:49:03.253263+0000","last_fullsized":"2024-03-07T03:49:03.253263+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:04:07.468850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1640,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.256332+0000","last_change":"2024-03-07T03:22:13.272925+0000","last_active":"2024-03-07T03:49:03.256332+0000","last_peered":"2024-03-07T03:49:03.256332+0000","last_clean":"2024-03-07T03:49:03.256332+0000","last_became_active":"2024-03-07T03:22:13.272599+0000","last_became_peered":"2024-03-07T03:22:13.272599+0000","last_unstale":"2024-03-07T03:49:03.256332+0000","last_undegraded":"2024-03-07T03:49:03.256332+0000","last_fullsized":"2024-03-07T03:49:03.256332+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:40:25.777060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1069,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.962410+0000","last_change":"2024-03-07T03:19:38.535826+0000","last_active":"2024-03-07T03:48:58.962410+0000","last_peered":"2024-03-07T03:48:58.962410+0000","last_clean":"2024-03-07T03:48:58.962410+0000","last_became_active":"2024-03-07T03:19:38.534837+0000","last_became_peered":"2024-03-07T03:19:38.534837+0000","last_unstale":"2024-03-07T03:48:58.962410+0000","last_undegraded":"2024-03-07T03:48:58.962410+0000","last_fullsized":"2024-03-07T03:48:58.962410+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:15:51.581170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1539,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253403+0000","last_change":"2024-03-07T03:22:13.269477+0000","last_active":"2024-03-07T03:49:03.253403+0000","last_peered":"2024-03-07T03:49:03.253403+0000","last_clean":"2024-03-07T03:49:03.253403+0000","last_became_active":"2024-03-07T03:22:13.269253+0000","last_became_peered":"2024-03-07T03:22:13.269253+0000","last_unstale":"2024-03-07T03:49:03.253403+0000","last_undegraded":"2024-03-07T03:49:03.253403+0000","last_fullsized":"2024-03-07T03:49:03.253403+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:58:44.512937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1644,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.254338+0000","last_change":"2024-03-07T03:22:13.285202+0000","last_active":"2024-03-07T03:49:03.254338+0000","last_peered":"2024-03-07T03:49:03.254338+0000","last_clean":"2024-03-07T03:49:03.254338+0000","last_became_active":"2024-03-07T03:22:13.284711+0000","last_became_peered":"2024-03-07T03:22:13.284711+0000","last_unstale":"2024-03-07T03:49:03.254338+0000","last_undegraded":"2024-03-07T03:49:03.254338+0000","last_fullsized":"2024-03-07T03:49:03.254338+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:11:51.417811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1625,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.267011+0000","last_change":"2024-03-07T03:22:41.913007+0000","last_active":"2024-03-07T03:49:03.267011+0000","last_peered":"2024-03-07T03:49:03.267011+0000","last_clean":"2024-03-07T03:49:03.267011+0000","last_became_active":"2024-03-07T03:22:41.912732+0000","last_became_peered":"2024-03-07T03:22:41.912732+0000","last_unstale":"2024-03-07T03:49:03.267011+0000","last_undegraded":"2024-03-07T03:49:03.267011+0000","last_fullsized":"2024-03-07T03:49:03.267011+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:07:37.955743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1532,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.255605+0000","last_change":"2024-03-07T03:22:41.933063+0000","last_active":"2024-03-07T03:49:03.255605+0000","last_peered":"2024-03-07T03:49:03.255605+0000","last_clean":"2024-03-07T03:49:03.255605+0000","last_became_active":"2024-03-07T03:22:41.932741+0000","last_became_peered":"2024-03-07T03:22:41.932741+0000","last_unstale":"2024-03-07T03:49:03.255605+0000","last_undegraded":"2024-03-07T03:49:03.255605+0000","last_fullsized":"2024-03-07T03:49:03.255605+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:18:09.308129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":1540,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.160736+0000","last_change":"2024-03-07T03:22:41.911676+0000","last_active":"2024-03-07T03:49:03.160736+0000","last_peered":"2024-03-07T03:49:03.160736+0000","last_clean":"2024-03-07T03:49:03.160736+0000","last_became_active":"2024-03-07T03:22:41.911271+0000","last_became_peered":"2024-03-07T03:22:41.911271+0000","last_unstale":"2024-03-07T03:49:03.160736+0000","last_undegraded":"2024-03-07T03:49:03.160736+0000","last_fullsized":"2024-03-07T03:49:03.160736+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:00:50.586457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":65,"num_read_kb":68,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2196,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201827,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":112120,"kb_used_data":12972,"kb_used_omap":1,"kb_used_meta":99134,"kb_avail":93633032,"statfs":{"total":95995035648,"available":95880224768,"internally_reserved":0,"allocated":13283328,"data_stored":8522565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":101513483},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725111,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":105824,"kb_used_data":12372,"kb_used_omap":1,"kb_used_meta":93438,"kb_avail":93639328,"statfs":{"total":95995035648,"available":95886671872,"internally_reserved":0,"allocated":12668928,"data_stored":7927877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":95680779},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":655,"seq":2813203579216,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":62380,"kb_used_data":12388,"kb_used_omap":71,"kb_used_meta":49912,"kb_avail":93682772,"statfs":{"total":95995035648,"available":95931158528,"internally_reserved":0,"allocated":12685312,"data_stored":7932139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73151,"internal_metadata":51110465},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":636,"seq":2731599200604,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":92960,"kb_used_data":12372,"kb_used_omap":74,"kb_used_meta":80501,"kb_avail":93652192,"statfs":{"total":95995035648,"available":95899844608,"internally_reserved":0,"allocated":12668928,"data_stored":7927853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":75974,"internal_metadata":82433850},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463469,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":116204,"kb_used_data":12960,"kb_used_omap":1,"kb_used_meta":103230,"kb_avail":93628948,"statfs":{"total":95995035648,"available":95876042752,"internally_reserved":0,"allocated":13271040,"data_stored":8522419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":105707787},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":483,"seq":2074469204347,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76708,"kb_used_data":12952,"kb_used_omap":62,"kb_used_meta":63681,"kb_avail":93668444,"statfs":{"total":95995035648,"available":95916486656,"internally_reserved":0,"allocated":13262848,"data_stored":8518209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":63655,"internal_metadata":65210201},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":518,"seq":2224793059701,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":80280,"kb_used_data":12364,"kb_used_omap":53,"kb_used_meta":67850,"kb_avail":93664872,"statfs":{"total":95995035648,"available":95912828928,"internally_reserved":0,"allocated":12660736,"data_stored":7927237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55112,"internal_metadata":69478584},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":648,"seq":2783138808150,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76380,"kb_used_data":12368,"kb_used_omap":64,"kb_used_meta":63935,"kb_avail":93668772,"statfs":{"total":95995035648,"available":95916822528,"internally_reserved":0,"allocated":12664832,"data_stored":7927327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":66149,"internal_metadata":65469851},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1299,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":897,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-07T03:50:33.593 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph pg dump --format=json 2024-03-07T03:50:34.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:33 smithi145 bash[79878]: audit 2024-03-07T03:50:32.736129+0000 mgr.y (mgr.34107) 1161 : audit 0 from='client.34992 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:50:34.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:33 smithi145 bash[82329]: audit 2024-03-07T03:50:32.736129+0000 mgr.y (mgr.34107) 1161 : audit 0 from='client.34992 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:50:34.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:33 smithi189 bash[66969]: audit 2024-03-07T03:50:32.736129+0000 mgr.y (mgr.34107) 1161 : audit 0 from='client.34992 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:50:35.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:34 smithi145 bash[79878]: cluster 2024-03-07T03:50:33.699235+0000 mgr.y (mgr.34107) 1162 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:35.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:34 smithi145 bash[82329]: cluster 2024-03-07T03:50:33.699235+0000 mgr.y (mgr.34107) 1162 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:35.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:34 smithi189 bash[66969]: cluster 2024-03-07T03:50:33.699235+0000 mgr.y (mgr.34107) 1162 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:37.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:36 smithi145 bash[79878]: cluster 2024-03-07T03:50:35.699986+0000 mgr.y (mgr.34107) 1163 : cluster 0 pgmap v1888: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:37.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:36 smithi145 bash[79878]: audit 2024-03-07T03:50:36.250979+0000 mon.a (mon.0) 1977 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:50:37.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:36 smithi145 bash[82329]: cluster 2024-03-07T03:50:35.699986+0000 mgr.y (mgr.34107) 1163 : cluster 0 pgmap v1888: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:37.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:36 smithi145 bash[82329]: audit 2024-03-07T03:50:36.250979+0000 mon.a (mon.0) 1977 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:50:37.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:36 smithi189 bash[66969]: cluster 2024-03-07T03:50:35.699986+0000 mgr.y (mgr.34107) 1163 : cluster 0 pgmap v1888: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:37.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:36 smithi189 bash[66969]: audit 2024-03-07T03:50:36.250979+0000 mon.a (mon.0) 1977 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:50:38.331 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:50:39.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:38 smithi145 bash[79878]: cluster 2024-03-07T03:50:37.700980+0000 mgr.y (mgr.34107) 1164 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:39.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:38 smithi145 bash[82329]: cluster 2024-03-07T03:50:37.700980+0000 mgr.y (mgr.34107) 1164 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:39.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:38 smithi189 bash[66969]: cluster 2024-03-07T03:50:37.700980+0000 mgr.y (mgr.34107) 1164 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:41.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:40 smithi145 bash[79878]: cluster 2024-03-07T03:50:39.701854+0000 mgr.y (mgr.34107) 1165 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:41.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:40 smithi145 bash[82329]: cluster 2024-03-07T03:50:39.701854+0000 mgr.y (mgr.34107) 1165 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:41.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:40 smithi189 bash[66969]: cluster 2024-03-07T03:50:39.701854+0000 mgr.y (mgr.34107) 1165 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:41.500 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:50:41.501 INFO:teuthology.orchestra.run.smithi145.stderr:dumped all 2024-03-07T03:50:42.297 INFO:teuthology.orchestra.run.smithi145.stdout:{"pg_ready":true,"pg_map":{"version":1890,"stamp":"2024-03-07T03:50:39.701271+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":565,"num_read_kb":992,"num_write":300,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":722856,"kb_used_data":100748,"kb_used_omap":331,"kb_used_meta":621684,"kb_avail":749238360,"statfs":{"total":767960285184,"available":767220080640,"internally_reserved":0,"allocated":103165952,"data_stored":65205626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":339384,"internal_metadata":636605000},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005408"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":1522,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.155075+0000","last_change":"2024-03-07T03:22:41.925126+0000","last_active":"2024-03-07T03:49:03.155075+0000","last_peered":"2024-03-07T03:49:03.155075+0000","last_clean":"2024-03-07T03:49:03.155075+0000","last_became_active":"2024-03-07T03:22:41.924495+0000","last_became_peered":"2024-03-07T03:22:41.924495+0000","last_unstale":"2024-03-07T03:49:03.155075+0000","last_undegraded":"2024-03-07T03:49:03.155075+0000","last_fullsized":"2024-03-07T03:49:03.155075+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:47:40.875079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":1654,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.256624+0000","last_change":"2024-03-07T03:21:43.302554+0000","last_active":"2024-03-07T03:49:03.256624+0000","last_peered":"2024-03-07T03:49:03.256624+0000","last_clean":"2024-03-07T03:49:03.256624+0000","last_became_active":"2024-03-07T03:21:43.301987+0000","last_became_peered":"2024-03-07T03:21:43.301987+0000","last_unstale":"2024-03-07T03:49:03.256624+0000","last_undegraded":"2024-03-07T03:49:03.256624+0000","last_fullsized":"2024-03-07T03:49:03.256624+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:57:41.827315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1538,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.256236+0000","last_change":"2024-03-07T03:22:13.255033+0000","last_active":"2024-03-07T03:49:03.256236+0000","last_peered":"2024-03-07T03:49:03.256236+0000","last_clean":"2024-03-07T03:49:03.256236+0000","last_became_active":"2024-03-07T03:22:13.254227+0000","last_became_peered":"2024-03-07T03:22:13.254227+0000","last_unstale":"2024-03-07T03:49:03.256236+0000","last_undegraded":"2024-03-07T03:49:03.256236+0000","last_fullsized":"2024-03-07T03:49:03.256236+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:49:30.698453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":1530,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.257190+0000","last_change":"2024-03-07T03:22:13.280263+0000","last_active":"2024-03-07T03:49:03.257190+0000","last_peered":"2024-03-07T03:49:03.257190+0000","last_clean":"2024-03-07T03:49:03.257190+0000","last_became_active":"2024-03-07T03:22:13.279394+0000","last_became_peered":"2024-03-07T03:22:13.279394+0000","last_unstale":"2024-03-07T03:49:03.257190+0000","last_undegraded":"2024-03-07T03:49:03.257190+0000","last_fullsized":"2024-03-07T03:49:03.257190+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:31:37.602107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1537,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.161211+0000","last_change":"2024-03-07T03:22:41.953431+0000","last_active":"2024-03-07T03:49:03.161211+0000","last_peered":"2024-03-07T03:49:03.161211+0000","last_clean":"2024-03-07T03:49:03.161211+0000","last_became_active":"2024-03-07T03:22:41.946313+0000","last_became_peered":"2024-03-07T03:22:41.946313+0000","last_unstale":"2024-03-07T03:49:03.161211+0000","last_undegraded":"2024-03-07T03:49:03.161211+0000","last_fullsized":"2024-03-07T03:49:03.161211+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:47:23.279612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":1066,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.144779+0000","last_change":"2024-03-07T03:22:41.935909+0000","last_active":"2024-03-07T03:48:58.144779+0000","last_peered":"2024-03-07T03:48:58.144779+0000","last_clean":"2024-03-07T03:48:58.144779+0000","last_became_active":"2024-03-07T03:22:41.935561+0000","last_became_peered":"2024-03-07T03:22:41.935561+0000","last_unstale":"2024-03-07T03:48:58.144779+0000","last_undegraded":"2024-03-07T03:48:58.144779+0000","last_fullsized":"2024-03-07T03:48:58.144779+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:09:48.621152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1537,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.157195+0000","last_change":"2024-03-07T03:22:41.931999+0000","last_active":"2024-03-07T03:49:03.157195+0000","last_peered":"2024-03-07T03:49:03.157195+0000","last_clean":"2024-03-07T03:49:03.157195+0000","last_became_active":"2024-03-07T03:22:41.931590+0000","last_became_peered":"2024-03-07T03:22:41.931590+0000","last_unstale":"2024-03-07T03:49:03.157195+0000","last_undegraded":"2024-03-07T03:49:03.157195+0000","last_fullsized":"2024-03-07T03:49:03.157195+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:46:30.686424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":1081,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.144521+0000","last_change":"2024-03-07T03:22:41.935688+0000","last_active":"2024-03-07T03:48:58.144521+0000","last_peered":"2024-03-07T03:48:58.144521+0000","last_clean":"2024-03-07T03:48:58.144521+0000","last_became_active":"2024-03-07T03:22:41.935333+0000","last_became_peered":"2024-03-07T03:22:41.935333+0000","last_unstale":"2024-03-07T03:48:58.144521+0000","last_undegraded":"2024-03-07T03:48:58.144521+0000","last_fullsized":"2024-03-07T03:48:58.144521+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:05:22.450810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1512,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.257799+0000","last_change":"2024-03-07T03:22:13.266027+0000","last_active":"2024-03-07T03:49:03.257799+0000","last_peered":"2024-03-07T03:49:03.257799+0000","last_clean":"2024-03-07T03:49:03.257799+0000","last_became_active":"2024-03-07T03:22:13.265741+0000","last_became_peered":"2024-03-07T03:22:13.265741+0000","last_unstale":"2024-03-07T03:49:03.257799+0000","last_undegraded":"2024-03-07T03:49:03.257799+0000","last_fullsized":"2024-03-07T03:49:03.257799+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:54:16.621812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":1645,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253149+0000","last_change":"2024-03-07T03:22:41.922062+0000","last_active":"2024-03-07T03:49:03.253149+0000","last_peered":"2024-03-07T03:49:03.253149+0000","last_clean":"2024-03-07T03:49:03.253149+0000","last_became_active":"2024-03-07T03:22:41.921629+0000","last_became_peered":"2024-03-07T03:22:41.921629+0000","last_unstale":"2024-03-07T03:49:03.253149+0000","last_undegraded":"2024-03-07T03:49:03.253149+0000","last_fullsized":"2024-03-07T03:49:03.253149+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:00:57.248696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1510,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.159076+0000","last_change":"2024-03-07T03:22:41.957347+0000","last_active":"2024-03-07T03:49:03.159076+0000","last_peered":"2024-03-07T03:49:03.159076+0000","last_clean":"2024-03-07T03:49:03.159076+0000","last_became_active":"2024-03-07T03:22:41.952518+0000","last_became_peered":"2024-03-07T03:22:41.952518+0000","last_unstale":"2024-03-07T03:49:03.159076+0000","last_undegraded":"2024-03-07T03:49:03.159076+0000","last_fullsized":"2024-03-07T03:49:03.159076+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:15:57.274707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":1656,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.252207+0000","last_change":"2024-03-07T03:22:41.921898+0000","last_active":"2024-03-07T03:49:03.252207+0000","last_peered":"2024-03-07T03:49:03.252207+0000","last_clean":"2024-03-07T03:49:03.252207+0000","last_became_active":"2024-03-07T03:22:41.921297+0000","last_became_peered":"2024-03-07T03:22:41.921297+0000","last_unstale":"2024-03-07T03:49:03.252207+0000","last_undegraded":"2024-03-07T03:49:03.252207+0000","last_fullsized":"2024-03-07T03:49:03.252207+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:09:06.393622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1075,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.151945+0000","last_change":"2024-03-07T03:21:43.304121+0000","last_active":"2024-03-07T03:49:03.151945+0000","last_peered":"2024-03-07T03:49:03.151945+0000","last_clean":"2024-03-07T03:49:03.151945+0000","last_became_active":"2024-03-07T03:21:43.302961+0000","last_became_peered":"2024-03-07T03:21:43.302961+0000","last_unstale":"2024-03-07T03:49:03.151945+0000","last_undegraded":"2024-03-07T03:49:03.151945+0000","last_fullsized":"2024-03-07T03:49:03.151945+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:13:38.289491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1527,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253421+0000","last_change":"2024-03-07T03:22:13.301668+0000","last_active":"2024-03-07T03:49:03.253421+0000","last_peered":"2024-03-07T03:49:03.253421+0000","last_clean":"2024-03-07T03:49:03.253421+0000","last_became_active":"2024-03-07T03:22:13.298895+0000","last_became_peered":"2024-03-07T03:22:13.298895+0000","last_unstale":"2024-03-07T03:49:03.253421+0000","last_undegraded":"2024-03-07T03:49:03.253421+0000","last_fullsized":"2024-03-07T03:49:03.253421+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:57:04.778318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1618,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.277039+0000","last_change":"2024-03-07T03:22:41.908219+0000","last_active":"2024-03-07T03:49:03.277039+0000","last_peered":"2024-03-07T03:49:03.277039+0000","last_clean":"2024-03-07T03:49:03.277039+0000","last_became_active":"2024-03-07T03:22:41.907902+0000","last_became_peered":"2024-03-07T03:22:41.907902+0000","last_unstale":"2024-03-07T03:49:03.277039+0000","last_undegraded":"2024-03-07T03:49:03.277039+0000","last_fullsized":"2024-03-07T03:49:03.277039+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:08:33.376548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":1494,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.256668+0000","last_change":"2024-03-07T03:22:13.292154+0000","last_active":"2024-03-07T03:49:03.256668+0000","last_peered":"2024-03-07T03:49:03.256668+0000","last_clean":"2024-03-07T03:49:03.256668+0000","last_became_active":"2024-03-07T03:22:13.291446+0000","last_became_peered":"2024-03-07T03:22:13.291446+0000","last_unstale":"2024-03-07T03:49:03.256668+0000","last_undegraded":"2024-03-07T03:49:03.256668+0000","last_fullsized":"2024-03-07T03:49:03.256668+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:16:03.766353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1635,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253066+0000","last_change":"2024-03-07T03:19:03.742695+0000","last_active":"2024-03-07T03:49:03.253066+0000","last_peered":"2024-03-07T03:49:03.253066+0000","last_clean":"2024-03-07T03:49:03.253066+0000","last_became_active":"2024-03-07T03:19:03.741911+0000","last_became_peered":"2024-03-07T03:19:03.741911+0000","last_unstale":"2024-03-07T03:49:03.253066+0000","last_undegraded":"2024-03-07T03:49:03.253066+0000","last_fullsized":"2024-03-07T03:49:03.253066+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:21:55.513877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1618,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.271691+0000","last_change":"2024-03-07T03:21:43.299149+0000","last_active":"2024-03-07T03:49:03.271691+0000","last_peered":"2024-03-07T03:49:03.271691+0000","last_clean":"2024-03-07T03:49:03.271691+0000","last_became_active":"2024-03-07T03:21:43.297777+0000","last_became_peered":"2024-03-07T03:21:43.297777+0000","last_unstale":"2024-03-07T03:49:03.271691+0000","last_undegraded":"2024-03-07T03:49:03.271691+0000","last_fullsized":"2024-03-07T03:49:03.271691+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:51:07.610242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1090,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.162609+0000","last_change":"2024-03-07T03:22:41.903154+0000","last_active":"2024-03-07T03:49:03.162609+0000","last_peered":"2024-03-07T03:49:03.162609+0000","last_clean":"2024-03-07T03:49:03.162609+0000","last_became_active":"2024-03-07T03:22:41.902822+0000","last_became_peered":"2024-03-07T03:22:41.902822+0000","last_unstale":"2024-03-07T03:49:03.162609+0000","last_undegraded":"2024-03-07T03:49:03.162609+0000","last_fullsized":"2024-03-07T03:49:03.162609+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:44:07.344147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1652,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.252498+0000","last_change":"2024-03-07T03:21:43.305697+0000","last_active":"2024-03-07T03:49:03.252498+0000","last_peered":"2024-03-07T03:49:03.252498+0000","last_clean":"2024-03-07T03:49:03.252498+0000","last_became_active":"2024-03-07T03:21:43.304986+0000","last_became_peered":"2024-03-07T03:21:43.304986+0000","last_unstale":"2024-03-07T03:49:03.252498+0000","last_undegraded":"2024-03-07T03:49:03.252498+0000","last_fullsized":"2024-03-07T03:49:03.252498+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:25:48.065258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1652,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.251610+0000","last_change":"2024-03-07T03:22:41.924664+0000","last_active":"2024-03-07T03:49:03.251610+0000","last_peered":"2024-03-07T03:49:03.251610+0000","last_clean":"2024-03-07T03:49:03.251610+0000","last_became_active":"2024-03-07T03:22:41.923356+0000","last_became_peered":"2024-03-07T03:22:41.923356+0000","last_unstale":"2024-03-07T03:49:03.251610+0000","last_undegraded":"2024-03-07T03:49:03.251610+0000","last_fullsized":"2024-03-07T03:49:03.251610+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:43:47.698590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1090,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.883631+0000","last_change":"2024-03-07T03:19:38.525605+0000","last_active":"2024-03-07T03:48:58.883631+0000","last_peered":"2024-03-07T03:48:58.883631+0000","last_clean":"2024-03-07T03:48:58.883631+0000","last_became_active":"2024-03-07T03:19:38.524575+0000","last_became_peered":"2024-03-07T03:19:38.524575+0000","last_unstale":"2024-03-07T03:48:58.883631+0000","last_undegraded":"2024-03-07T03:48:58.883631+0000","last_fullsized":"2024-03-07T03:48:58.883631+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:21:46.949046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1069,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.964954+0000","last_change":"2024-03-07T03:22:41.924297+0000","last_active":"2024-03-07T03:48:58.964954+0000","last_peered":"2024-03-07T03:48:58.964954+0000","last_clean":"2024-03-07T03:48:58.964954+0000","last_became_active":"2024-03-07T03:22:41.923679+0000","last_became_peered":"2024-03-07T03:22:41.923679+0000","last_unstale":"2024-03-07T03:48:58.964954+0000","last_undegraded":"2024-03-07T03:48:58.964954+0000","last_fullsized":"2024-03-07T03:48:58.964954+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:22:51.253808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1512,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.251735+0000","last_change":"2024-03-07T03:22:13.289010+0000","last_active":"2024-03-07T03:49:03.251735+0000","last_peered":"2024-03-07T03:49:03.251735+0000","last_clean":"2024-03-07T03:49:03.251735+0000","last_became_active":"2024-03-07T03:22:13.288665+0000","last_became_peered":"2024-03-07T03:22:13.288665+0000","last_unstale":"2024-03-07T03:49:03.251735+0000","last_undegraded":"2024-03-07T03:49:03.251735+0000","last_fullsized":"2024-03-07T03:49:03.251735+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:36:57.801158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"284'161","reported_seq":1440,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.156109+0000","last_change":"2024-03-07T03:19:03.750010+0000","last_active":"2024-03-07T03:49:03.156109+0000","last_peered":"2024-03-07T03:49:03.156109+0000","last_clean":"2024-03-07T03:49:03.156109+0000","last_became_active":"2024-03-07T03:19:03.749484+0000","last_became_peered":"2024-03-07T03:19:03.749484+0000","last_unstale":"2024-03-07T03:49:03.156109+0000","last_undegraded":"2024-03-07T03:49:03.156109+0000","last_fullsized":"2024-03-07T03:49:03.156109+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:01:00.212577+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:01:00.212577+0000","last_clean_scrub_stamp":"2024-03-07T03:01:00.212577+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:00:56.725739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1071,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.255232+0000","last_change":"2024-03-07T03:21:43.269933+0000","last_active":"2024-03-07T03:49:03.255232+0000","last_peered":"2024-03-07T03:49:03.255232+0000","last_clean":"2024-03-07T03:49:03.255232+0000","last_became_active":"2024-03-07T03:21:43.269289+0000","last_became_peered":"2024-03-07T03:21:43.269289+0000","last_unstale":"2024-03-07T03:49:03.255232+0000","last_undegraded":"2024-03-07T03:49:03.255232+0000","last_fullsized":"2024-03-07T03:49:03.255232+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:09:13.977042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1629,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.287624+0000","last_change":"2024-03-07T03:19:38.544337+0000","last_active":"2024-03-07T03:49:03.287624+0000","last_peered":"2024-03-07T03:49:03.287624+0000","last_clean":"2024-03-07T03:49:03.287624+0000","last_became_active":"2024-03-07T03:19:38.541610+0000","last_became_peered":"2024-03-07T03:19:38.541610+0000","last_unstale":"2024-03-07T03:49:03.287624+0000","last_undegraded":"2024-03-07T03:49:03.287624+0000","last_fullsized":"2024-03-07T03:49:03.287624+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:06:15.042560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1620,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.264891+0000","last_change":"2024-03-07T03:21:43.281389+0000","last_active":"2024-03-07T03:49:03.264891+0000","last_peered":"2024-03-07T03:49:03.264891+0000","last_clean":"2024-03-07T03:49:03.264891+0000","last_became_active":"2024-03-07T03:21:43.278292+0000","last_became_peered":"2024-03-07T03:21:43.278292+0000","last_unstale":"2024-03-07T03:49:03.264891+0000","last_undegraded":"2024-03-07T03:49:03.264891+0000","last_fullsized":"2024-03-07T03:49:03.264891+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:13:19.332308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1545,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.251961+0000","last_change":"2024-03-07T03:22:13.239633+0000","last_active":"2024-03-07T03:49:03.251961+0000","last_peered":"2024-03-07T03:49:03.251961+0000","last_clean":"2024-03-07T03:49:03.251961+0000","last_became_active":"2024-03-07T03:22:13.239322+0000","last_became_peered":"2024-03-07T03:22:13.239322+0000","last_unstale":"2024-03-07T03:49:03.251961+0000","last_undegraded":"2024-03-07T03:49:03.251961+0000","last_fullsized":"2024-03-07T03:49:03.251961+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:40:53.579559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1075,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.144133+0000","last_change":"2024-03-07T03:22:13.276342+0000","last_active":"2024-03-07T03:48:58.144133+0000","last_peered":"2024-03-07T03:48:58.144133+0000","last_clean":"2024-03-07T03:48:58.144133+0000","last_became_active":"2024-03-07T03:22:13.276074+0000","last_became_peered":"2024-03-07T03:22:13.276074+0000","last_unstale":"2024-03-07T03:48:58.144133+0000","last_undegraded":"2024-03-07T03:48:58.144133+0000","last_fullsized":"2024-03-07T03:48:58.144133+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:20:30.055957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1523,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.158042+0000","last_change":"2024-03-07T03:22:41.960515+0000","last_active":"2024-03-07T03:49:03.158042+0000","last_peered":"2024-03-07T03:49:03.158042+0000","last_clean":"2024-03-07T03:49:03.158042+0000","last_became_active":"2024-03-07T03:22:41.953664+0000","last_became_peered":"2024-03-07T03:22:41.953664+0000","last_unstale":"2024-03-07T03:49:03.158042+0000","last_undegraded":"2024-03-07T03:49:03.158042+0000","last_fullsized":"2024-03-07T03:49:03.158042+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:55:02.230660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1071,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.250765+0000","last_change":"2024-03-07T03:19:38.522178+0000","last_active":"2024-03-07T03:49:03.250765+0000","last_peered":"2024-03-07T03:49:03.250765+0000","last_clean":"2024-03-07T03:49:03.250765+0000","last_became_active":"2024-03-07T03:19:38.521276+0000","last_became_peered":"2024-03-07T03:19:38.521276+0000","last_unstale":"2024-03-07T03:49:03.250765+0000","last_undegraded":"2024-03-07T03:49:03.250765+0000","last_fullsized":"2024-03-07T03:49:03.250765+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:24:42.660567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1618,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.268690+0000","last_change":"2024-03-07T03:22:41.885534+0000","last_active":"2024-03-07T03:49:03.268690+0000","last_peered":"2024-03-07T03:49:03.268690+0000","last_clean":"2024-03-07T03:49:03.268690+0000","last_became_active":"2024-03-07T03:22:41.885340+0000","last_became_peered":"2024-03-07T03:22:41.885340+0000","last_unstale":"2024-03-07T03:49:03.268690+0000","last_undegraded":"2024-03-07T03:49:03.268690+0000","last_fullsized":"2024-03-07T03:49:03.268690+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:35:17.363850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1081,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.252766+0000","last_change":"2024-03-07T03:22:13.298793+0000","last_active":"2024-03-07T03:49:03.252766+0000","last_peered":"2024-03-07T03:49:03.252766+0000","last_clean":"2024-03-07T03:49:03.252766+0000","last_became_active":"2024-03-07T03:22:13.298480+0000","last_became_peered":"2024-03-07T03:22:13.298480+0000","last_unstale":"2024-03-07T03:49:03.252766+0000","last_undegraded":"2024-03-07T03:49:03.252766+0000","last_fullsized":"2024-03-07T03:49:03.252766+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:08:08.449442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1635,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.284706+0000","last_change":"2024-03-07T03:19:38.532550+0000","last_active":"2024-03-07T03:49:03.284706+0000","last_peered":"2024-03-07T03:49:03.284706+0000","last_clean":"2024-03-07T03:49:03.284706+0000","last_became_active":"2024-03-07T03:19:38.531243+0000","last_became_peered":"2024-03-07T03:19:38.531243+0000","last_unstale":"2024-03-07T03:49:03.284706+0000","last_undegraded":"2024-03-07T03:49:03.284706+0000","last_fullsized":"2024-03-07T03:49:03.284706+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:06:41.634794+0000","last_clean_scrub_stamp":"2024-03-07T03:06:41.634794+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:21:28.043587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1626,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.287462+0000","last_change":"2024-03-07T03:22:41.909258+0000","last_active":"2024-03-07T03:49:03.287462+0000","last_peered":"2024-03-07T03:49:03.287462+0000","last_clean":"2024-03-07T03:49:03.287462+0000","last_became_active":"2024-03-07T03:22:41.908514+0000","last_became_peered":"2024-03-07T03:22:41.908514+0000","last_unstale":"2024-03-07T03:49:03.287462+0000","last_undegraded":"2024-03-07T03:49:03.287462+0000","last_fullsized":"2024-03-07T03:49:03.287462+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:38:57.312991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1527,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253012+0000","last_change":"2024-03-07T03:22:13.304093+0000","last_active":"2024-03-07T03:49:03.253012+0000","last_peered":"2024-03-07T03:49:03.253012+0000","last_clean":"2024-03-07T03:49:03.253012+0000","last_became_active":"2024-03-07T03:22:13.301755+0000","last_became_peered":"2024-03-07T03:22:13.301755+0000","last_unstale":"2024-03-07T03:49:03.253012+0000","last_undegraded":"2024-03-07T03:49:03.253012+0000","last_fullsized":"2024-03-07T03:49:03.253012+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:05:55.915533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1514,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.167462+0000","last_change":"2024-03-07T03:22:41.926183+0000","last_active":"2024-03-07T03:49:03.167462+0000","last_peered":"2024-03-07T03:49:03.167462+0000","last_clean":"2024-03-07T03:49:03.167462+0000","last_became_active":"2024-03-07T03:22:41.925842+0000","last_became_peered":"2024-03-07T03:22:41.925842+0000","last_unstale":"2024-03-07T03:49:03.167462+0000","last_undegraded":"2024-03-07T03:49:03.167462+0000","last_fullsized":"2024-03-07T03:49:03.167462+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:55:23.626786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1525,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.153461+0000","last_change":"2024-03-07T03:22:41.956505+0000","last_active":"2024-03-07T03:49:03.153461+0000","last_peered":"2024-03-07T03:49:03.153461+0000","last_clean":"2024-03-07T03:49:03.153461+0000","last_became_active":"2024-03-07T03:22:41.943938+0000","last_became_peered":"2024-03-07T03:22:41.943938+0000","last_unstale":"2024-03-07T03:49:03.153461+0000","last_undegraded":"2024-03-07T03:49:03.153461+0000","last_fullsized":"2024-03-07T03:49:03.153461+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:37:04.758716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1517,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.165122+0000","last_change":"2024-03-07T03:22:41.914323+0000","last_active":"2024-03-07T03:49:03.165122+0000","last_peered":"2024-03-07T03:49:03.165122+0000","last_clean":"2024-03-07T03:49:03.165122+0000","last_became_active":"2024-03-07T03:22:41.913344+0000","last_became_peered":"2024-03-07T03:22:41.913344+0000","last_unstale":"2024-03-07T03:49:03.165122+0000","last_undegraded":"2024-03-07T03:49:03.165122+0000","last_fullsized":"2024-03-07T03:49:03.165122+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:30:59.510000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1085,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.149079+0000","last_change":"2024-03-07T03:22:41.922381+0000","last_active":"2024-03-07T03:49:03.149079+0000","last_peered":"2024-03-07T03:49:03.149079+0000","last_clean":"2024-03-07T03:49:03.149079+0000","last_became_active":"2024-03-07T03:22:41.921959+0000","last_became_peered":"2024-03-07T03:22:41.921959+0000","last_unstale":"2024-03-07T03:49:03.149079+0000","last_undegraded":"2024-03-07T03:49:03.149079+0000","last_fullsized":"2024-03-07T03:49:03.149079+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:32:00.451279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1520,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.158239+0000","last_change":"2024-03-07T03:22:41.957256+0000","last_active":"2024-03-07T03:49:03.158239+0000","last_peered":"2024-03-07T03:49:03.158239+0000","last_clean":"2024-03-07T03:49:03.158239+0000","last_became_active":"2024-03-07T03:22:41.947708+0000","last_became_peered":"2024-03-07T03:22:41.947708+0000","last_unstale":"2024-03-07T03:49:03.158239+0000","last_undegraded":"2024-03-07T03:49:03.158239+0000","last_fullsized":"2024-03-07T03:49:03.158239+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:49:19.355156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":1538,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253954+0000","last_change":"2024-03-07T03:22:41.931128+0000","last_active":"2024-03-07T03:49:03.253954+0000","last_peered":"2024-03-07T03:49:03.253954+0000","last_clean":"2024-03-07T03:49:03.253954+0000","last_became_active":"2024-03-07T03:22:41.930808+0000","last_became_peered":"2024-03-07T03:22:41.930808+0000","last_unstale":"2024-03-07T03:49:03.253954+0000","last_undegraded":"2024-03-07T03:49:03.253954+0000","last_fullsized":"2024-03-07T03:49:03.253954+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:01:52.246153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1665,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.252812+0000","last_change":"2024-03-07T03:22:41.923532+0000","last_active":"2024-03-07T03:49:03.252812+0000","last_peered":"2024-03-07T03:49:03.252812+0000","last_clean":"2024-03-07T03:49:03.252812+0000","last_became_active":"2024-03-07T03:22:41.923218+0000","last_became_peered":"2024-03-07T03:22:41.923218+0000","last_unstale":"2024-03-07T03:49:03.252812+0000","last_undegraded":"2024-03-07T03:49:03.252812+0000","last_fullsized":"2024-03-07T03:49:03.252812+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:13:19.255516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1064,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.252480+0000","last_change":"2024-03-07T03:22:13.302012+0000","last_active":"2024-03-07T03:49:03.252480+0000","last_peered":"2024-03-07T03:49:03.252480+0000","last_clean":"2024-03-07T03:49:03.252480+0000","last_became_active":"2024-03-07T03:22:13.301560+0000","last_became_peered":"2024-03-07T03:22:13.301560+0000","last_unstale":"2024-03-07T03:49:03.252480+0000","last_undegraded":"2024-03-07T03:49:03.252480+0000","last_fullsized":"2024-03-07T03:49:03.252480+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:43:14.805124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1069,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.148656+0000","last_change":"2024-03-07T03:19:38.535906+0000","last_active":"2024-03-07T03:49:03.148656+0000","last_peered":"2024-03-07T03:49:03.148656+0000","last_clean":"2024-03-07T03:49:03.148656+0000","last_became_active":"2024-03-07T03:19:38.535036+0000","last_became_peered":"2024-03-07T03:19:38.535036+0000","last_unstale":"2024-03-07T03:49:03.148656+0000","last_undegraded":"2024-03-07T03:49:03.148656+0000","last_fullsized":"2024-03-07T03:49:03.148656+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:21:05.129278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":1055,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.162376+0000","last_change":"2024-03-07T03:08:49.421752+0000","last_active":"2024-03-07T03:49:03.162376+0000","last_peered":"2024-03-07T03:49:03.162376+0000","last_clean":"2024-03-07T03:49:03.162376+0000","last_became_active":"2024-03-07T03:08:49.421165+0000","last_became_peered":"2024-03-07T03:08:49.421165+0000","last_unstale":"2024-03-07T03:49:03.162376+0000","last_undegraded":"2024-03-07T03:49:03.162376+0000","last_fullsized":"2024-03-07T03:49:03.162376+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:57:59.044174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1539,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.159223+0000","last_change":"2024-03-07T03:22:41.931071+0000","last_active":"2024-03-07T03:49:03.159223+0000","last_peered":"2024-03-07T03:49:03.159223+0000","last_clean":"2024-03-07T03:49:03.159223+0000","last_became_active":"2024-03-07T03:22:41.930732+0000","last_became_peered":"2024-03-07T03:22:41.930732+0000","last_unstale":"2024-03-07T03:49:03.159223+0000","last_undegraded":"2024-03-07T03:49:03.159223+0000","last_fullsized":"2024-03-07T03:49:03.159223+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:44:46.520072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":1087,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.163516+0000","last_change":"2024-03-07T03:22:41.904040+0000","last_active":"2024-03-07T03:49:03.163516+0000","last_peered":"2024-03-07T03:49:03.163516+0000","last_clean":"2024-03-07T03:49:03.163516+0000","last_became_active":"2024-03-07T03:22:41.903638+0000","last_became_peered":"2024-03-07T03:22:41.903638+0000","last_unstale":"2024-03-07T03:49:03.163516+0000","last_undegraded":"2024-03-07T03:49:03.163516+0000","last_fullsized":"2024-03-07T03:49:03.163516+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:24:34.364981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1049,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.250205+0000","last_change":"2024-03-07T03:08:50.110958+0000","last_active":"2024-03-07T03:49:03.250205+0000","last_peered":"2024-03-07T03:49:03.250205+0000","last_clean":"2024-03-07T03:49:03.250205+0000","last_became_active":"2024-03-07T03:08:50.110066+0000","last_became_peered":"2024-03-07T03:08:50.110066+0000","last_unstale":"2024-03-07T03:49:03.250205+0000","last_undegraded":"2024-03-07T03:49:03.250205+0000","last_fullsized":"2024-03-07T03:49:03.250205+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:57:28.837409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":1085,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.166054+0000","last_change":"2024-03-07T03:22:41.906951+0000","last_active":"2024-03-07T03:49:03.166054+0000","last_peered":"2024-03-07T03:49:03.166054+0000","last_clean":"2024-03-07T03:49:03.166054+0000","last_became_active":"2024-03-07T03:22:41.906609+0000","last_became_peered":"2024-03-07T03:22:41.906609+0000","last_unstale":"2024-03-07T03:49:03.166054+0000","last_undegraded":"2024-03-07T03:49:03.166054+0000","last_fullsized":"2024-03-07T03:49:03.166054+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:13:25.180063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1522,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.255641+0000","last_change":"2024-03-07T03:22:13.298545+0000","last_active":"2024-03-07T03:49:03.255641+0000","last_peered":"2024-03-07T03:49:03.255641+0000","last_clean":"2024-03-07T03:49:03.255641+0000","last_became_active":"2024-03-07T03:22:13.298225+0000","last_became_peered":"2024-03-07T03:22:13.298225+0000","last_unstale":"2024-03-07T03:49:03.255641+0000","last_undegraded":"2024-03-07T03:49:03.255641+0000","last_fullsized":"2024-03-07T03:49:03.255641+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:10:15.575412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":1084,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.963172+0000","last_change":"2024-03-07T03:19:03.727952+0000","last_active":"2024-03-07T03:48:58.963172+0000","last_peered":"2024-03-07T03:48:58.963172+0000","last_clean":"2024-03-07T03:48:58.963172+0000","last_became_active":"2024-03-07T03:19:03.727379+0000","last_became_peered":"2024-03-07T03:19:03.727379+0000","last_unstale":"2024-03-07T03:48:58.963172+0000","last_undegraded":"2024-03-07T03:48:58.963172+0000","last_fullsized":"2024-03-07T03:48:58.963172+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:02:49.830245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1641,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.251895+0000","last_change":"2024-03-07T03:22:13.285040+0000","last_active":"2024-03-07T03:49:03.251895+0000","last_peered":"2024-03-07T03:49:03.251895+0000","last_clean":"2024-03-07T03:49:03.251895+0000","last_became_active":"2024-03-07T03:22:13.284262+0000","last_became_peered":"2024-03-07T03:22:13.284262+0000","last_unstale":"2024-03-07T03:49:03.251895+0000","last_undegraded":"2024-03-07T03:49:03.251895+0000","last_fullsized":"2024-03-07T03:49:03.251895+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:20:44.532634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1067,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.884958+0000","last_change":"2024-03-07T03:22:41.932835+0000","last_active":"2024-03-07T03:48:58.884958+0000","last_peered":"2024-03-07T03:48:58.884958+0000","last_clean":"2024-03-07T03:48:58.884958+0000","last_became_active":"2024-03-07T03:22:41.932544+0000","last_became_peered":"2024-03-07T03:22:41.932544+0000","last_unstale":"2024-03-07T03:48:58.884958+0000","last_undegraded":"2024-03-07T03:48:58.884958+0000","last_fullsized":"2024-03-07T03:48:58.884958+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:57:31.879866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1062,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.252741+0000","last_change":"2024-03-07T03:22:13.307826+0000","last_active":"2024-03-07T03:49:03.252741+0000","last_peered":"2024-03-07T03:49:03.252741+0000","last_clean":"2024-03-07T03:49:03.252741+0000","last_became_active":"2024-03-07T03:22:13.306153+0000","last_became_peered":"2024-03-07T03:22:13.306153+0000","last_unstale":"2024-03-07T03:49:03.252741+0000","last_undegraded":"2024-03-07T03:49:03.252741+0000","last_fullsized":"2024-03-07T03:49:03.252741+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:28:23.430883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1640,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.254200+0000","last_change":"2024-03-07T03:22:13.284663+0000","last_active":"2024-03-07T03:49:03.254200+0000","last_peered":"2024-03-07T03:49:03.254200+0000","last_clean":"2024-03-07T03:49:03.254200+0000","last_became_active":"2024-03-07T03:22:13.284296+0000","last_became_peered":"2024-03-07T03:22:13.284296+0000","last_unstale":"2024-03-07T03:49:03.254200+0000","last_undegraded":"2024-03-07T03:49:03.254200+0000","last_fullsized":"2024-03-07T03:49:03.254200+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:27:27.139954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1523,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.261804+0000","last_change":"2024-03-07T03:22:13.300789+0000","last_active":"2024-03-07T03:49:03.261804+0000","last_peered":"2024-03-07T03:49:03.261804+0000","last_clean":"2024-03-07T03:49:03.261804+0000","last_became_active":"2024-03-07T03:22:13.295435+0000","last_became_peered":"2024-03-07T03:22:13.295435+0000","last_unstale":"2024-03-07T03:49:03.261804+0000","last_undegraded":"2024-03-07T03:49:03.261804+0000","last_fullsized":"2024-03-07T03:49:03.261804+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:24:33.504833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":1086,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-03-07T03:48:58.885558+0000","last_change":"2024-03-07T03:22:41.932597+0000","last_active":"2024-03-07T03:48:58.885558+0000","last_peered":"2024-03-07T03:48:58.885558+0000","last_clean":"2024-03-07T03:48:58.885558+0000","last_became_active":"2024-03-07T03:22:41.931421+0000","last_became_peered":"2024-03-07T03:22:41.931421+0000","last_unstale":"2024-03-07T03:48:58.885558+0000","last_undegraded":"2024-03-07T03:48:58.885558+0000","last_fullsized":"2024-03-07T03:48:58.885558+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:43:47.536816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":8,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1522,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.157666+0000","last_change":"2024-03-07T03:22:41.929992+0000","last_active":"2024-03-07T03:49:03.157666+0000","last_peered":"2024-03-07T03:49:03.157666+0000","last_clean":"2024-03-07T03:49:03.157666+0000","last_became_active":"2024-03-07T03:22:41.928923+0000","last_became_peered":"2024-03-07T03:22:41.928923+0000","last_unstale":"2024-03-07T03:49:03.157666+0000","last_undegraded":"2024-03-07T03:49:03.157666+0000","last_fullsized":"2024-03-07T03:49:03.157666+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:03:09.890598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"338'23","reported_seq":1511,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.158385+0000","last_change":"2024-03-07T03:22:41.958079+0000","last_active":"2024-03-07T03:49:03.158385+0000","last_peered":"2024-03-07T03:49:03.158385+0000","last_clean":"2024-03-07T03:49:03.158385+0000","last_became_active":"2024-03-07T03:22:41.949612+0000","last_became_peered":"2024-03-07T03:22:41.949612+0000","last_unstale":"2024-03-07T03:49:03.158385+0000","last_undegraded":"2024-03-07T03:49:03.158385+0000","last_fullsized":"2024-03-07T03:49:03.158385+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:05:27.967329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1617,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.268009+0000","last_change":"2024-03-07T03:21:43.279937+0000","last_active":"2024-03-07T03:49:03.268009+0000","last_peered":"2024-03-07T03:49:03.268009+0000","last_clean":"2024-03-07T03:49:03.268009+0000","last_became_active":"2024-03-07T03:21:43.279154+0000","last_became_peered":"2024-03-07T03:21:43.279154+0000","last_unstale":"2024-03-07T03:49:03.268009+0000","last_undegraded":"2024-03-07T03:49:03.268009+0000","last_fullsized":"2024-03-07T03:49:03.268009+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:54:02.755057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":1088,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.162781+0000","last_change":"2024-03-07T03:22:41.904810+0000","last_active":"2024-03-07T03:49:03.162781+0000","last_peered":"2024-03-07T03:49:03.162781+0000","last_clean":"2024-03-07T03:49:03.162781+0000","last_became_active":"2024-03-07T03:22:41.903580+0000","last_became_peered":"2024-03-07T03:22:41.903580+0000","last_unstale":"2024-03-07T03:49:03.162781+0000","last_undegraded":"2024-03-07T03:49:03.162781+0000","last_fullsized":"2024-03-07T03:49:03.162781+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:24:19.771759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1509,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.256753+0000","last_change":"2024-03-07T03:22:13.291868+0000","last_active":"2024-03-07T03:49:03.256753+0000","last_peered":"2024-03-07T03:49:03.256753+0000","last_clean":"2024-03-07T03:49:03.256753+0000","last_became_active":"2024-03-07T03:22:13.291433+0000","last_became_peered":"2024-03-07T03:22:13.291433+0000","last_unstale":"2024-03-07T03:49:03.256753+0000","last_undegraded":"2024-03-07T03:49:03.256753+0000","last_fullsized":"2024-03-07T03:49:03.256753+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:37:17.893023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1063,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.252175+0000","last_change":"2024-03-07T03:19:38.536795+0000","last_active":"2024-03-07T03:49:03.252175+0000","last_peered":"2024-03-07T03:49:03.252175+0000","last_clean":"2024-03-07T03:49:03.252175+0000","last_became_active":"2024-03-07T03:19:38.535022+0000","last_became_peered":"2024-03-07T03:19:38.535022+0000","last_unstale":"2024-03-07T03:49:03.252175+0000","last_undegraded":"2024-03-07T03:49:03.252175+0000","last_fullsized":"2024-03-07T03:49:03.252175+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:35:50.615645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1524,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253263+0000","last_change":"2024-03-07T03:22:13.280490+0000","last_active":"2024-03-07T03:49:03.253263+0000","last_peered":"2024-03-07T03:49:03.253263+0000","last_clean":"2024-03-07T03:49:03.253263+0000","last_became_active":"2024-03-07T03:22:13.280062+0000","last_became_peered":"2024-03-07T03:22:13.280062+0000","last_unstale":"2024-03-07T03:49:03.253263+0000","last_undegraded":"2024-03-07T03:49:03.253263+0000","last_fullsized":"2024-03-07T03:49:03.253263+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:04:07.468850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1640,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.256332+0000","last_change":"2024-03-07T03:22:13.272925+0000","last_active":"2024-03-07T03:49:03.256332+0000","last_peered":"2024-03-07T03:49:03.256332+0000","last_clean":"2024-03-07T03:49:03.256332+0000","last_became_active":"2024-03-07T03:22:13.272599+0000","last_became_peered":"2024-03-07T03:22:13.272599+0000","last_unstale":"2024-03-07T03:49:03.256332+0000","last_undegraded":"2024-03-07T03:49:03.256332+0000","last_fullsized":"2024-03-07T03:49:03.256332+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:40:25.777060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1070,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.151731+0000","last_change":"2024-03-07T03:19:38.535826+0000","last_active":"2024-03-07T03:49:03.151731+0000","last_peered":"2024-03-07T03:49:03.151731+0000","last_clean":"2024-03-07T03:49:03.151731+0000","last_became_active":"2024-03-07T03:19:38.534837+0000","last_became_peered":"2024-03-07T03:19:38.534837+0000","last_unstale":"2024-03-07T03:49:03.151731+0000","last_undegraded":"2024-03-07T03:49:03.151731+0000","last_fullsized":"2024-03-07T03:49:03.151731+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:15:51.581170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1539,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.253403+0000","last_change":"2024-03-07T03:22:13.269477+0000","last_active":"2024-03-07T03:49:03.253403+0000","last_peered":"2024-03-07T03:49:03.253403+0000","last_clean":"2024-03-07T03:49:03.253403+0000","last_became_active":"2024-03-07T03:22:13.269253+0000","last_became_peered":"2024-03-07T03:22:13.269253+0000","last_unstale":"2024-03-07T03:49:03.253403+0000","last_undegraded":"2024-03-07T03:49:03.253403+0000","last_fullsized":"2024-03-07T03:49:03.253403+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:58:44.512937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1644,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.254338+0000","last_change":"2024-03-07T03:22:13.285202+0000","last_active":"2024-03-07T03:49:03.254338+0000","last_peered":"2024-03-07T03:49:03.254338+0000","last_clean":"2024-03-07T03:49:03.254338+0000","last_became_active":"2024-03-07T03:22:13.284711+0000","last_became_peered":"2024-03-07T03:22:13.284711+0000","last_unstale":"2024-03-07T03:49:03.254338+0000","last_undegraded":"2024-03-07T03:49:03.254338+0000","last_fullsized":"2024-03-07T03:49:03.254338+0000","mapping_epoch":648,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":649,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:11:51.417811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1625,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.267011+0000","last_change":"2024-03-07T03:22:41.913007+0000","last_active":"2024-03-07T03:49:03.267011+0000","last_peered":"2024-03-07T03:49:03.267011+0000","last_clean":"2024-03-07T03:49:03.267011+0000","last_became_active":"2024-03-07T03:22:41.912732+0000","last_became_peered":"2024-03-07T03:22:41.912732+0000","last_unstale":"2024-03-07T03:49:03.267011+0000","last_undegraded":"2024-03-07T03:49:03.267011+0000","last_fullsized":"2024-03-07T03:49:03.267011+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:07:37.955743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1532,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.255605+0000","last_change":"2024-03-07T03:22:41.933063+0000","last_active":"2024-03-07T03:49:03.255605+0000","last_peered":"2024-03-07T03:49:03.255605+0000","last_clean":"2024-03-07T03:49:03.255605+0000","last_became_active":"2024-03-07T03:22:41.932741+0000","last_became_peered":"2024-03-07T03:22:41.932741+0000","last_unstale":"2024-03-07T03:49:03.255605+0000","last_undegraded":"2024-03-07T03:49:03.255605+0000","last_fullsized":"2024-03-07T03:49:03.255605+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:49.370704+0000","last_clean_scrub_stamp":"2024-03-07T03:08:49.370704+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:18:09.308129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":1540,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:49:03.160736+0000","last_change":"2024-03-07T03:22:41.911676+0000","last_active":"2024-03-07T03:49:03.160736+0000","last_peered":"2024-03-07T03:49:03.160736+0000","last_clean":"2024-03-07T03:49:03.160736+0000","last_became_active":"2024-03-07T03:22:41.911271+0000","last_became_peered":"2024-03-07T03:22:41.911271+0000","last_unstale":"2024-03-07T03:49:03.160736+0000","last_undegraded":"2024-03-07T03:49:03.160736+0000","last_fullsized":"2024-03-07T03:49:03.160736+0000","mapping_epoch":655,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":656,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:08:48.359089+0000","last_clean_scrub_stamp":"2024-03-07T03:08:48.359089+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:00:50.586457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":65,"num_read_kb":68,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2196,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201829,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":112120,"kb_used_data":12972,"kb_used_omap":1,"kb_used_meta":99134,"kb_avail":93633032,"statfs":{"total":95995035648,"available":95880224768,"internally_reserved":0,"allocated":13283328,"data_stored":8522565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":101513483},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725113,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":105824,"kb_used_data":12372,"kb_used_omap":1,"kb_used_meta":93438,"kb_avail":93639328,"statfs":{"total":95995035648,"available":95886671872,"internally_reserved":0,"allocated":12668928,"data_stored":7927877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":95680779},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":655,"seq":2813203579217,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":62380,"kb_used_data":12388,"kb_used_omap":71,"kb_used_meta":49912,"kb_avail":93682772,"statfs":{"total":95995035648,"available":95931158528,"internally_reserved":0,"allocated":12685312,"data_stored":7932139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":73151,"internal_metadata":51110465},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":636,"seq":2731599200605,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":92960,"kb_used_data":12372,"kb_used_omap":74,"kb_used_meta":80501,"kb_avail":93652192,"statfs":{"total":95995035648,"available":95899844608,"internally_reserved":0,"allocated":12668928,"data_stored":7927853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":75974,"internal_metadata":82433850},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463471,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":116204,"kb_used_data":12960,"kb_used_omap":1,"kb_used_meta":103230,"kb_avail":93628948,"statfs":{"total":95995035648,"available":95876042752,"internally_reserved":0,"allocated":13271040,"data_stored":8522419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":105707787},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":483,"seq":2074469204349,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76708,"kb_used_data":12952,"kb_used_omap":62,"kb_used_meta":63681,"kb_avail":93668444,"statfs":{"total":95995035648,"available":95916486656,"internally_reserved":0,"allocated":13262848,"data_stored":8518209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":63655,"internal_metadata":65210201},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":518,"seq":2224793059702,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":80280,"kb_used_data":12364,"kb_used_omap":53,"kb_used_meta":67850,"kb_avail":93664872,"statfs":{"total":95995035648,"available":95912828928,"internally_reserved":0,"allocated":12660736,"data_stored":7927237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55112,"internal_metadata":69478584},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":648,"seq":2783138808151,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76380,"kb_used_data":12368,"kb_used_omap":64,"kb_used_meta":63935,"kb_avail":93668772,"statfs":{"total":95995035648,"available":95916822528,"internally_reserved":0,"allocated":12664832,"data_stored":7927327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":66149,"internal_metadata":65469851},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1299,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":897,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-07T03:50:42.301 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-07T03:50:42.301 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-07T03:50:42.301 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-07T03:50:42.301 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:50:42.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:42 smithi145 bash[79878]: audit 2024-03-07T03:50:41.439307+0000 mon.a (mon.0) 1978 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:42.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:42 smithi145 bash[79878]: audit 2024-03-07T03:50:41.445026+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:42.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:42 smithi145 bash[79878]: audit 2024-03-07T03:50:41.491911+0000 mgr.y (mgr.34107) 1166 : audit 0 from='client.44915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:50:42.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:42 smithi145 bash[79878]: cluster 2024-03-07T03:50:41.702739+0000 mgr.y (mgr.34107) 1167 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:42.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:42 smithi145 bash[79878]: audit 2024-03-07T03:50:42.302600+0000 mon.a (mon.0) 1980 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:42.569 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:42 smithi145 bash[79878]: audit 2024-03-07T03:50:42.313378+0000 mon.a (mon.0) 1981 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:42.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:42 smithi145 bash[82329]: audit 2024-03-07T03:50:41.439307+0000 mon.a (mon.0) 1978 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:42.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:42 smithi145 bash[82329]: audit 2024-03-07T03:50:41.445026+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:42.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:42 smithi145 bash[82329]: audit 2024-03-07T03:50:41.491911+0000 mgr.y (mgr.34107) 1166 : audit 0 from='client.44915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:50:42.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:42 smithi145 bash[82329]: cluster 2024-03-07T03:50:41.702739+0000 mgr.y (mgr.34107) 1167 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:42.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:42 smithi145 bash[82329]: audit 2024-03-07T03:50:42.302600+0000 mon.a (mon.0) 1980 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:42.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:42 smithi145 bash[82329]: audit 2024-03-07T03:50:42.313378+0000 mon.a (mon.0) 1981 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:42.589 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:50:42 smithi189 bash[59814]: ts=2024-03-07T03:50:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:50:42.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:42 smithi189 bash[66969]: audit 2024-03-07T03:50:41.439307+0000 mon.a (mon.0) 1978 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:42.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:42 smithi189 bash[66969]: audit 2024-03-07T03:50:41.445026+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:42.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:42 smithi189 bash[66969]: audit 2024-03-07T03:50:41.491911+0000 mgr.y (mgr.34107) 1166 : audit 0 from='client.44915 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:50:42.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:42 smithi189 bash[66969]: cluster 2024-03-07T03:50:41.702739+0000 mgr.y (mgr.34107) 1167 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:42.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:42 smithi189 bash[66969]: audit 2024-03-07T03:50:42.302600+0000 mon.a (mon.0) 1980 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:42.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:42 smithi189 bash[66969]: audit 2024-03-07T03:50:42.313378+0000 mon.a (mon.0) 1981 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:43.068 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:50:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:50:42] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.43.0" 2024-03-07T03:50:43.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:43 smithi145 bash[79878]: audit 2024-03-07T03:50:42.988106+0000 mon.a (mon.0) 1982 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:50:43.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:43 smithi145 bash[79878]: audit 2024-03-07T03:50:42.990077+0000 mon.a (mon.0) 1983 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:50:43.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:43 smithi145 bash[79878]: audit 2024-03-07T03:50:43.000734+0000 mon.a (mon.0) 1984 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:43.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:43 smithi145 bash[82329]: audit 2024-03-07T03:50:42.988106+0000 mon.a (mon.0) 1982 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:50:43.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:43 smithi145 bash[82329]: audit 2024-03-07T03:50:42.990077+0000 mon.a (mon.0) 1983 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:50:43.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:43 smithi145 bash[82329]: audit 2024-03-07T03:50:43.000734+0000 mon.a (mon.0) 1984 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:43.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:43 smithi189 bash[66969]: audit 2024-03-07T03:50:42.988106+0000 mon.a (mon.0) 1982 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:50:43.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:43 smithi189 bash[66969]: audit 2024-03-07T03:50:42.990077+0000 mon.a (mon.0) 1983 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:50:43.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:43 smithi189 bash[66969]: audit 2024-03-07T03:50:43.000734+0000 mon.a (mon.0) 1984 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:50:44.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:44 smithi145 bash[79878]: cluster 2024-03-07T03:50:43.703657+0000 mgr.y (mgr.34107) 1168 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:44.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:44 smithi145 bash[82329]: cluster 2024-03-07T03:50:43.703657+0000 mgr.y (mgr.34107) 1168 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:44.839 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:44 smithi189 bash[66969]: cluster 2024-03-07T03:50:43.703657+0000 mgr.y (mgr.34107) 1168 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:47.039 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:50:47.060 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:46 smithi145 bash[79878]: cluster 2024-03-07T03:50:45.704448+0000 mgr.y (mgr.34107) 1169 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:47.060 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:46 smithi145 bash[82329]: cluster 2024-03-07T03:50:45.704448+0000 mgr.y (mgr.34107) 1169 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:47.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:46 smithi189 bash[66969]: cluster 2024-03-07T03:50:45.704448+0000 mgr.y (mgr.34107) 1169 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:48.068 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:47 smithi145 bash[79878]: audit 2024-03-07T03:50:47.686195+0000 mon.a (mon.0) 1985 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:48.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:47 smithi145 bash[82329]: audit 2024-03-07T03:50:47.686195+0000 mon.a (mon.0) 1985 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:48.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:47 smithi189 bash[66969]: audit 2024-03-07T03:50:47.686195+0000 mon.a (mon.0) 1985 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:49.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:48 smithi145 bash[79878]: cluster 2024-03-07T03:50:47.705278+0000 mgr.y (mgr.34107) 1170 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:49.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:48 smithi145 bash[82329]: cluster 2024-03-07T03:50:47.705278+0000 mgr.y (mgr.34107) 1170 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:49.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:48 smithi189 bash[66969]: cluster 2024-03-07T03:50:47.705278+0000 mgr.y (mgr.34107) 1170 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:50.275 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:50:50.275 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:50:50.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:50 smithi145 bash[79878]: cluster 2024-03-07T03:50:49.706111+0000 mgr.y (mgr.34107) 1171 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:50.819 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:50 smithi145 bash[79878]: audit 2024-03-07T03:50:50.277676+0000 mon.a (mon.0) 1986 : audit 0 from='client.? 172.21.15.145:0/1598126838' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:50:50.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:50 smithi145 bash[82329]: cluster 2024-03-07T03:50:49.706111+0000 mgr.y (mgr.34107) 1171 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:50.819 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:50 smithi145 bash[82329]: audit 2024-03-07T03:50:50.277676+0000 mon.a (mon.0) 1986 : audit 0 from='client.? 172.21.15.145:0/1598126838' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:50:51.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:50 smithi189 bash[66969]: cluster 2024-03-07T03:50:49.706111+0000 mgr.y (mgr.34107) 1171 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:51.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:50 smithi189 bash[66969]: audit 2024-03-07T03:50:50.277676+0000 mon.a (mon.0) 1986 : audit 0 from='client.? 172.21.15.145:0/1598126838' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:50:52.591 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:50:52 smithi189 bash[59814]: ts=2024-03-07T03:50:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:50:53.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:52 smithi145 bash[79878]: cluster 2024-03-07T03:50:51.706985+0000 mgr.y (mgr.34107) 1172 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:53.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:50:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:50:52] "GET /metrics HTTP/1.1" 200 34153 "" "Prometheus/2.43.0" 2024-03-07T03:50:53.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:52 smithi145 bash[82329]: cluster 2024-03-07T03:50:51.706985+0000 mgr.y (mgr.34107) 1172 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:53.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:52 smithi189 bash[66969]: cluster 2024-03-07T03:50:51.706985+0000 mgr.y (mgr.34107) 1172 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:54.065 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:50:55.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:54 smithi145 bash[82329]: cluster 2024-03-07T03:50:53.707858+0000 mgr.y (mgr.34107) 1173 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:55.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:54 smithi145 bash[79878]: cluster 2024-03-07T03:50:53.707858+0000 mgr.y (mgr.34107) 1173 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:55.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:54 smithi189 bash[66969]: cluster 2024-03-07T03:50:53.707858+0000 mgr.y (mgr.34107) 1173 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:57.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:56 smithi145 bash[82329]: cluster 2024-03-07T03:50:55.708651+0000 mgr.y (mgr.34107) 1174 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:57.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:56 smithi145 bash[79878]: cluster 2024-03-07T03:50:55.708651+0000 mgr.y (mgr.34107) 1174 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:57.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:56 smithi189 bash[66969]: cluster 2024-03-07T03:50:55.708651+0000 mgr.y (mgr.34107) 1174 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:58.802 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:50:59.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:50:58 smithi145 bash[79878]: cluster 2024-03-07T03:50:57.709588+0000 mgr.y (mgr.34107) 1175 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:59.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:50:58 smithi145 bash[82329]: cluster 2024-03-07T03:50:57.709588+0000 mgr.y (mgr.34107) 1175 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:50:59.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:50:58 smithi189 bash[66969]: cluster 2024-03-07T03:50:57.709588+0000 mgr.y (mgr.34107) 1175 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:01.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:00 smithi145 bash[79878]: cluster 2024-03-07T03:50:59.710432+0000 mgr.y (mgr.34107) 1176 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:01.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:00 smithi145 bash[82329]: cluster 2024-03-07T03:50:59.710432+0000 mgr.y (mgr.34107) 1176 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:01.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:00 smithi189 bash[66969]: cluster 2024-03-07T03:50:59.710432+0000 mgr.y (mgr.34107) 1176 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:02.149 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:51:02.149 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:51:02.589 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:51:02 smithi189 bash[59814]: ts=2024-03-07T03:51:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:51:03.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:51:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:51:02] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2024-03-07T03:51:03.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:02 smithi145 bash[79878]: cluster 2024-03-07T03:51:01.711316+0000 mgr.y (mgr.34107) 1177 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:03.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:02 smithi145 bash[79878]: audit 2024-03-07T03:51:02.151350+0000 mon.a (mon.0) 1987 : audit 0 from='client.? 172.21.15.145:0/1225141324' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:03.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:02 smithi145 bash[79878]: audit 2024-03-07T03:51:02.686634+0000 mon.a (mon.0) 1988 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:03.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:02 smithi145 bash[82329]: cluster 2024-03-07T03:51:01.711316+0000 mgr.y (mgr.34107) 1177 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:03.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:02 smithi145 bash[82329]: audit 2024-03-07T03:51:02.151350+0000 mon.a (mon.0) 1987 : audit 0 from='client.? 172.21.15.145:0/1225141324' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:03.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:02 smithi145 bash[82329]: audit 2024-03-07T03:51:02.686634+0000 mon.a (mon.0) 1988 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:03.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:02 smithi189 bash[66969]: cluster 2024-03-07T03:51:01.711316+0000 mgr.y (mgr.34107) 1177 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:03.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:02 smithi189 bash[66969]: audit 2024-03-07T03:51:02.151350+0000 mon.a (mon.0) 1987 : audit 0 from='client.? 172.21.15.145:0/1225141324' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:03.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:02 smithi189 bash[66969]: audit 2024-03-07T03:51:02.686634+0000 mon.a (mon.0) 1988 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:05.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:04 smithi145 bash[79878]: cluster 2024-03-07T03:51:03.712208+0000 mgr.y (mgr.34107) 1178 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:05.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:04 smithi145 bash[82329]: cluster 2024-03-07T03:51:03.712208+0000 mgr.y (mgr.34107) 1178 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:05.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:04 smithi189 bash[66969]: cluster 2024-03-07T03:51:03.712208+0000 mgr.y (mgr.34107) 1178 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:05.893 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:51:07.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:06 smithi145 bash[79878]: cluster 2024-03-07T03:51:05.712943+0000 mgr.y (mgr.34107) 1179 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:07.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:06 smithi145 bash[82329]: cluster 2024-03-07T03:51:05.712943+0000 mgr.y (mgr.34107) 1179 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:07.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:06 smithi189 bash[66969]: cluster 2024-03-07T03:51:05.712943+0000 mgr.y (mgr.34107) 1179 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:09.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:08 smithi145 bash[79878]: cluster 2024-03-07T03:51:07.713826+0000 mgr.y (mgr.34107) 1180 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:09.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:08 smithi145 bash[82329]: cluster 2024-03-07T03:51:07.713826+0000 mgr.y (mgr.34107) 1180 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:09.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:08 smithi189 bash[66969]: cluster 2024-03-07T03:51:07.713826+0000 mgr.y (mgr.34107) 1180 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:10.629 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:11.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:10 smithi145 bash[79878]: cluster 2024-03-07T03:51:09.714716+0000 mgr.y (mgr.34107) 1181 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:11.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:10 smithi145 bash[82329]: cluster 2024-03-07T03:51:09.714716+0000 mgr.y (mgr.34107) 1181 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:11.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:10 smithi189 bash[66969]: cluster 2024-03-07T03:51:09.714716+0000 mgr.y (mgr.34107) 1181 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:12.589 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:51:12 smithi189 bash[59814]: ts=2024-03-07T03:51:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:51:13.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:12 smithi145 bash[79878]: cluster 2024-03-07T03:51:11.715650+0000 mgr.y (mgr.34107) 1182 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:13.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:51:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:51:12] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2024-03-07T03:51:13.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:12 smithi145 bash[82329]: cluster 2024-03-07T03:51:11.715650+0000 mgr.y (mgr.34107) 1182 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:13.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:12 smithi189 bash[66969]: cluster 2024-03-07T03:51:11.715650+0000 mgr.y (mgr.34107) 1182 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:14.261 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:51:14.261 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:51:15.088 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:14 smithi145 bash[82329]: cluster 2024-03-07T03:51:13.716573+0000 mgr.y (mgr.34107) 1183 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:15.088 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:14 smithi145 bash[82329]: audit 2024-03-07T03:51:14.263804+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.145:0/3297402260' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:15.089 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:14 smithi145 bash[79878]: cluster 2024-03-07T03:51:13.716573+0000 mgr.y (mgr.34107) 1183 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:15.089 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:14 smithi145 bash[79878]: audit 2024-03-07T03:51:14.263804+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.145:0/3297402260' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:15.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:14 smithi189 bash[66969]: cluster 2024-03-07T03:51:13.716573+0000 mgr.y (mgr.34107) 1183 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:15.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:14 smithi189 bash[66969]: audit 2024-03-07T03:51:14.263804+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.145:0/3297402260' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:17.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:16 smithi189 bash[66969]: cluster 2024-03-07T03:51:15.717337+0000 mgr.y (mgr.34107) 1184 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:17.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:16 smithi145 bash[79878]: cluster 2024-03-07T03:51:15.717337+0000 mgr.y (mgr.34107) 1184 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:17.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:16 smithi145 bash[82329]: cluster 2024-03-07T03:51:15.717337+0000 mgr.y (mgr.34107) 1184 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:18.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:17 smithi189 bash[66969]: audit 2024-03-07T03:51:17.687152+0000 mon.a (mon.0) 1989 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:18.091 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:51:18.102 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:17 smithi145 bash[79878]: audit 2024-03-07T03:51:17.687152+0000 mon.a (mon.0) 1989 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:18.103 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:17 smithi145 bash[82329]: audit 2024-03-07T03:51:17.687152+0000 mon.a (mon.0) 1989 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:19.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:18 smithi189 bash[66969]: cluster 2024-03-07T03:51:17.726404+0000 mgr.y (mgr.34107) 1185 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:19.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:18 smithi145 bash[79878]: cluster 2024-03-07T03:51:17.726404+0000 mgr.y (mgr.34107) 1185 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:19.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:18 smithi145 bash[82329]: cluster 2024-03-07T03:51:17.726404+0000 mgr.y (mgr.34107) 1185 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:21.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:20 smithi145 bash[79878]: cluster 2024-03-07T03:51:19.727277+0000 mgr.y (mgr.34107) 1186 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:21.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:20 smithi145 bash[82329]: cluster 2024-03-07T03:51:19.727277+0000 mgr.y (mgr.34107) 1186 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:21.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:20 smithi189 bash[66969]: cluster 2024-03-07T03:51:19.727277+0000 mgr.y (mgr.34107) 1186 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:22.589 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:51:22 smithi189 bash[59814]: ts=2024-03-07T03:51:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:51:22.840 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:23.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:22 smithi145 bash[79878]: cluster 2024-03-07T03:51:21.728244+0000 mgr.y (mgr.34107) 1187 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:23.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:51:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:51:22] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2024-03-07T03:51:23.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:22 smithi145 bash[82329]: cluster 2024-03-07T03:51:21.728244+0000 mgr.y (mgr.34107) 1187 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:23.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:22 smithi189 bash[66969]: cluster 2024-03-07T03:51:21.728244+0000 mgr.y (mgr.34107) 1187 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:25.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:24 smithi145 bash[79878]: cluster 2024-03-07T03:51:23.729107+0000 mgr.y (mgr.34107) 1188 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:25.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:24 smithi145 bash[82329]: cluster 2024-03-07T03:51:23.729107+0000 mgr.y (mgr.34107) 1188 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:25.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:24 smithi189 bash[66969]: cluster 2024-03-07T03:51:23.729107+0000 mgr.y (mgr.34107) 1188 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:26.186 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:51:26.187 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:51:27.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:26 smithi145 bash[79878]: cluster 2024-03-07T03:51:25.729880+0000 mgr.y (mgr.34107) 1189 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:27.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:26 smithi145 bash[79878]: audit 2024-03-07T03:51:26.188894+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.145:0/708116067' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:27.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:26 smithi145 bash[82329]: cluster 2024-03-07T03:51:25.729880+0000 mgr.y (mgr.34107) 1189 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:27.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:26 smithi145 bash[82329]: audit 2024-03-07T03:51:26.188894+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.145:0/708116067' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:27.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:26 smithi189 bash[66969]: cluster 2024-03-07T03:51:25.729880+0000 mgr.y (mgr.34107) 1189 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:27.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:26 smithi189 bash[66969]: audit 2024-03-07T03:51:26.188894+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.145:0/708116067' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:29.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:28 smithi145 bash[79878]: cluster 2024-03-07T03:51:27.730909+0000 mgr.y (mgr.34107) 1190 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:29.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:28 smithi145 bash[82329]: cluster 2024-03-07T03:51:27.730909+0000 mgr.y (mgr.34107) 1190 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:29.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:28 smithi189 bash[66969]: cluster 2024-03-07T03:51:27.730909+0000 mgr.y (mgr.34107) 1190 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:29.959 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:51:31.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:30 smithi145 bash[79878]: cluster 2024-03-07T03:51:29.731915+0000 mgr.y (mgr.34107) 1191 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:31.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:30 smithi145 bash[82329]: cluster 2024-03-07T03:51:29.731915+0000 mgr.y (mgr.34107) 1191 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:31.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:30 smithi189 bash[66969]: cluster 2024-03-07T03:51:29.731915+0000 mgr.y (mgr.34107) 1191 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:32.589 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:51:32 smithi189 bash[59814]: ts=2024-03-07T03:51:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:51:33.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:32 smithi145 bash[79878]: cluster 2024-03-07T03:51:31.733004+0000 mgr.y (mgr.34107) 1192 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:33.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:32 smithi145 bash[79878]: audit 2024-03-07T03:51:32.686793+0000 mon.a (mon.0) 1990 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:33.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:51:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:51:32] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2024-03-07T03:51:33.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:32 smithi145 bash[82329]: cluster 2024-03-07T03:51:31.733004+0000 mgr.y (mgr.34107) 1192 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:33.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:32 smithi145 bash[82329]: audit 2024-03-07T03:51:32.686793+0000 mon.a (mon.0) 1990 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:33.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:32 smithi189 bash[66969]: cluster 2024-03-07T03:51:31.733004+0000 mgr.y (mgr.34107) 1192 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:33.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:32 smithi189 bash[66969]: audit 2024-03-07T03:51:32.686793+0000 mon.a (mon.0) 1990 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:34.699 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:35.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:34 smithi145 bash[79878]: cluster 2024-03-07T03:51:33.733858+0000 mgr.y (mgr.34107) 1193 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:35.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:34 smithi145 bash[82329]: cluster 2024-03-07T03:51:33.733858+0000 mgr.y (mgr.34107) 1193 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:35.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:34 smithi189 bash[66969]: cluster 2024-03-07T03:51:33.733858+0000 mgr.y (mgr.34107) 1193 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:37.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:36 smithi145 bash[79878]: cluster 2024-03-07T03:51:35.734488+0000 mgr.y (mgr.34107) 1194 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:37.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:36 smithi145 bash[82329]: cluster 2024-03-07T03:51:35.734488+0000 mgr.y (mgr.34107) 1194 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:37.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:36 smithi189 bash[66969]: cluster 2024-03-07T03:51:35.734488+0000 mgr.y (mgr.34107) 1194 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:37.926 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:51:37.927 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:51:39.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:38 smithi145 bash[79878]: cluster 2024-03-07T03:51:37.735329+0000 mgr.y (mgr.34107) 1195 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:39.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:38 smithi145 bash[79878]: audit 2024-03-07T03:51:37.928697+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.145:0/4211863767' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:39.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:38 smithi145 bash[82329]: cluster 2024-03-07T03:51:37.735329+0000 mgr.y (mgr.34107) 1195 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:39.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:38 smithi145 bash[82329]: audit 2024-03-07T03:51:37.928697+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.145:0/4211863767' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:39.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:38 smithi189 bash[66969]: cluster 2024-03-07T03:51:37.735329+0000 mgr.y (mgr.34107) 1195 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:39.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:38 smithi189 bash[66969]: audit 2024-03-07T03:51:37.928697+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.145:0/4211863767' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:41.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:40 smithi145 bash[79878]: cluster 2024-03-07T03:51:39.736164+0000 mgr.y (mgr.34107) 1196 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:41.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:40 smithi145 bash[82329]: cluster 2024-03-07T03:51:39.736164+0000 mgr.y (mgr.34107) 1196 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:41.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:40 smithi189 bash[66969]: cluster 2024-03-07T03:51:39.736164+0000 mgr.y (mgr.34107) 1196 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:41.785 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:51:42.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:51:42 smithi189 bash[59814]: ts=2024-03-07T03:51:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:51:43.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:42 smithi145 bash[79878]: cluster 2024-03-07T03:51:41.737083+0000 mgr.y (mgr.34107) 1197 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:43.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:42 smithi145 bash[82329]: cluster 2024-03-07T03:51:41.737083+0000 mgr.y (mgr.34107) 1197 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:43.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:51:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:51:42] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2024-03-07T03:51:43.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:42 smithi189 bash[66969]: cluster 2024-03-07T03:51:41.737083+0000 mgr.y (mgr.34107) 1197 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:44.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:43 smithi145 bash[79878]: audit 2024-03-07T03:51:43.094355+0000 mon.a (mon.0) 1991 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:51:44.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:43 smithi145 bash[82329]: audit 2024-03-07T03:51:43.094355+0000 mon.a (mon.0) 1991 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:51:44.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:43 smithi189 bash[66969]: audit 2024-03-07T03:51:43.094355+0000 mon.a (mon.0) 1991 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:51:45.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:44 smithi145 bash[79878]: cluster 2024-03-07T03:51:43.738013+0000 mgr.y (mgr.34107) 1198 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:45.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:44 smithi145 bash[82329]: cluster 2024-03-07T03:51:43.738013+0000 mgr.y (mgr.34107) 1198 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:45.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:44 smithi189 bash[66969]: cluster 2024-03-07T03:51:43.738013+0000 mgr.y (mgr.34107) 1198 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:46.524 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:47.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:46 smithi145 bash[79878]: cluster 2024-03-07T03:51:45.738796+0000 mgr.y (mgr.34107) 1199 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:47.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:46 smithi145 bash[82329]: cluster 2024-03-07T03:51:45.738796+0000 mgr.y (mgr.34107) 1199 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:47.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:46 smithi189 bash[66969]: cluster 2024-03-07T03:51:45.738796+0000 mgr.y (mgr.34107) 1199 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:48.073 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:47 smithi145 bash[79878]: audit 2024-03-07T03:51:47.686888+0000 mon.a (mon.0) 1992 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:48.073 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:47 smithi145 bash[82329]: audit 2024-03-07T03:51:47.686888+0000 mon.a (mon.0) 1992 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:48.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:47 smithi189 bash[66969]: audit 2024-03-07T03:51:47.686888+0000 mon.a (mon.0) 1992 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:49.542 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:49 smithi145 bash[82329]: cluster 2024-03-07T03:51:47.739415+0000 mgr.y (mgr.34107) 1200 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:49.542 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:49 smithi145 bash[82329]: audit 2024-03-07T03:51:48.331573+0000 mon.a (mon.0) 1993 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:49.542 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:49 smithi145 bash[82329]: audit 2024-03-07T03:51:48.339861+0000 mon.a (mon.0) 1994 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:49.542 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:49 smithi145 bash[82329]: audit 2024-03-07T03:51:49.146785+0000 mon.a (mon.0) 1995 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:49.542 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:49 smithi145 bash[82329]: audit 2024-03-07T03:51:49.160521+0000 mon.a (mon.0) 1996 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:49.542 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:49 smithi145 bash[79878]: cluster 2024-03-07T03:51:47.739415+0000 mgr.y (mgr.34107) 1200 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:49.542 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:49 smithi145 bash[79878]: audit 2024-03-07T03:51:48.331573+0000 mon.a (mon.0) 1993 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:49.542 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:49 smithi145 bash[79878]: audit 2024-03-07T03:51:48.339861+0000 mon.a (mon.0) 1994 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:49.542 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:49 smithi145 bash[79878]: audit 2024-03-07T03:51:49.146785+0000 mon.a (mon.0) 1995 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:49.543 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:49 smithi145 bash[79878]: audit 2024-03-07T03:51:49.160521+0000 mon.a (mon.0) 1996 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:49 smithi189 bash[66969]: cluster 2024-03-07T03:51:47.739415+0000 mgr.y (mgr.34107) 1200 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:49 smithi189 bash[66969]: audit 2024-03-07T03:51:48.331573+0000 mon.a (mon.0) 1993 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:49 smithi189 bash[66969]: audit 2024-03-07T03:51:48.339861+0000 mon.a (mon.0) 1994 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:49.589 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:49 smithi189 bash[66969]: audit 2024-03-07T03:51:49.146785+0000 mon.a (mon.0) 1995 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:49.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:49 smithi189 bash[66969]: audit 2024-03-07T03:51:49.160521+0000 mon.a (mon.0) 1996 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:49.938 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:51:49.938 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:51:50.208 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:50 smithi145 bash[79878]: audit 2024-03-07T03:51:49.731142+0000 mon.a (mon.0) 1997 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:51:50.209 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:50 smithi145 bash[79878]: audit 2024-03-07T03:51:49.732463+0000 mon.a (mon.0) 1998 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:51:50.209 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:50 smithi145 bash[79878]: audit 2024-03-07T03:51:49.739853+0000 mon.a (mon.0) 1999 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:50.209 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:50 smithi145 bash[79878]: cluster 2024-03-07T03:51:49.740368+0000 mgr.y (mgr.34107) 1201 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:50.209 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:50 smithi145 bash[79878]: audit 2024-03-07T03:51:49.940017+0000 mon.a (mon.0) 2000 : audit 0 from='client.? 172.21.15.145:0/2315642556' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:50.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:50 smithi145 bash[82329]: audit 2024-03-07T03:51:49.731142+0000 mon.a (mon.0) 1997 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:51:50.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:50 smithi145 bash[82329]: audit 2024-03-07T03:51:49.732463+0000 mon.a (mon.0) 1998 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:51:50.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:50 smithi145 bash[82329]: audit 2024-03-07T03:51:49.739853+0000 mon.a (mon.0) 1999 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:50.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:50 smithi145 bash[82329]: cluster 2024-03-07T03:51:49.740368+0000 mgr.y (mgr.34107) 1201 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:50.209 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:50 smithi145 bash[82329]: audit 2024-03-07T03:51:49.940017+0000 mon.a (mon.0) 2000 : audit 0 from='client.? 172.21.15.145:0/2315642556' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:50.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:50 smithi189 bash[66969]: audit 2024-03-07T03:51:49.731142+0000 mon.a (mon.0) 1997 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:51:50.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:50 smithi189 bash[66969]: audit 2024-03-07T03:51:49.732463+0000 mon.a (mon.0) 1998 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:51:50.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:50 smithi189 bash[66969]: audit 2024-03-07T03:51:49.739853+0000 mon.a (mon.0) 1999 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:51:50.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:50 smithi189 bash[66969]: cluster 2024-03-07T03:51:49.740368+0000 mgr.y (mgr.34107) 1201 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:50.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:50 smithi189 bash[66969]: audit 2024-03-07T03:51:49.940017+0000 mon.a (mon.0) 2000 : audit 0 from='client.? 172.21.15.145:0/2315642556' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:51:52.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:51:52 smithi189 bash[59814]: ts=2024-03-07T03:51:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:51:53.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:52 smithi145 bash[79878]: cluster 2024-03-07T03:51:51.741349+0000 mgr.y (mgr.34107) 1202 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:53.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:51:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:51:52] "GET /metrics HTTP/1.1" 200 34153 "" "Prometheus/2.43.0" 2024-03-07T03:51:53.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:52 smithi145 bash[82329]: cluster 2024-03-07T03:51:51.741349+0000 mgr.y (mgr.34107) 1202 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:53.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:52 smithi189 bash[66969]: cluster 2024-03-07T03:51:51.741349+0000 mgr.y (mgr.34107) 1202 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:53.781 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:51:55.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:54 smithi145 bash[79878]: cluster 2024-03-07T03:51:53.742107+0000 mgr.y (mgr.34107) 1203 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:55.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:54 smithi145 bash[82329]: cluster 2024-03-07T03:51:53.742107+0000 mgr.y (mgr.34107) 1203 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:55.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:54 smithi189 bash[66969]: cluster 2024-03-07T03:51:53.742107+0000 mgr.y (mgr.34107) 1203 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:57.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:56 smithi145 bash[79878]: cluster 2024-03-07T03:51:55.742812+0000 mgr.y (mgr.34107) 1204 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:57.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:56 smithi145 bash[82329]: cluster 2024-03-07T03:51:55.742812+0000 mgr.y (mgr.34107) 1204 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:57.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:56 smithi189 bash[66969]: cluster 2024-03-07T03:51:55.742812+0000 mgr.y (mgr.34107) 1204 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:58.518 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:59.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:51:58 smithi145 bash[79878]: cluster 2024-03-07T03:51:57.743666+0000 mgr.y (mgr.34107) 1205 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:59.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:51:58 smithi145 bash[82329]: cluster 2024-03-07T03:51:57.743666+0000 mgr.y (mgr.34107) 1205 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:59.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:51:58 smithi189 bash[66969]: cluster 2024-03-07T03:51:57.743666+0000 mgr.y (mgr.34107) 1205 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:01.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:00 smithi145 bash[79878]: cluster 2024-03-07T03:51:59.744379+0000 mgr.y (mgr.34107) 1206 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:01.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:00 smithi145 bash[82329]: cluster 2024-03-07T03:51:59.744379+0000 mgr.y (mgr.34107) 1206 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:01.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:00 smithi189 bash[66969]: cluster 2024-03-07T03:51:59.744379+0000 mgr.y (mgr.34107) 1206 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:01.966 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:52:01.967 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:52:02.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:52:02 smithi189 bash[59814]: ts=2024-03-07T03:52:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:52:03.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:52:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:52:02] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2024-03-07T03:52:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:02 smithi145 bash[79878]: cluster 2024-03-07T03:52:01.744889+0000 mgr.y (mgr.34107) 1207 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:02 smithi145 bash[79878]: audit 2024-03-07T03:52:01.968593+0000 mon.c (mon.2) 60 : audit 0 from='client.? 172.21.15.145:0/1593653610' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:03.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:02 smithi145 bash[79878]: audit 2024-03-07T03:52:02.688187+0000 mon.a (mon.0) 2001 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:03.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:02 smithi145 bash[82329]: cluster 2024-03-07T03:52:01.744889+0000 mgr.y (mgr.34107) 1207 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:03.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:02 smithi145 bash[82329]: audit 2024-03-07T03:52:01.968593+0000 mon.c (mon.2) 60 : audit 0 from='client.? 172.21.15.145:0/1593653610' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:03.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:02 smithi145 bash[82329]: audit 2024-03-07T03:52:02.688187+0000 mon.a (mon.0) 2001 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:03.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:02 smithi189 bash[66969]: cluster 2024-03-07T03:52:01.744889+0000 mgr.y (mgr.34107) 1207 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:03.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:02 smithi189 bash[66969]: audit 2024-03-07T03:52:01.968593+0000 mon.c (mon.2) 60 : audit 0 from='client.? 172.21.15.145:0/1593653610' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:03.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:02 smithi189 bash[66969]: audit 2024-03-07T03:52:02.688187+0000 mon.a (mon.0) 2001 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:05.089 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:04 smithi189 bash[66969]: cluster 2024-03-07T03:52:03.745639+0000 mgr.y (mgr.34107) 1208 : cluster 0 pgmap v1932: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:05.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:04 smithi145 bash[79878]: cluster 2024-03-07T03:52:03.745639+0000 mgr.y (mgr.34107) 1208 : cluster 0 pgmap v1932: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:05.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:04 smithi145 bash[82329]: cluster 2024-03-07T03:52:03.745639+0000 mgr.y (mgr.34107) 1208 : cluster 0 pgmap v1932: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:05.732 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:52:07.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:06 smithi189 bash[66969]: cluster 2024-03-07T03:52:05.746456+0000 mgr.y (mgr.34107) 1209 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:07.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:06 smithi145 bash[79878]: cluster 2024-03-07T03:52:05.746456+0000 mgr.y (mgr.34107) 1209 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:07.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:06 smithi145 bash[82329]: cluster 2024-03-07T03:52:05.746456+0000 mgr.y (mgr.34107) 1209 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:09.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:08 smithi189 bash[66969]: cluster 2024-03-07T03:52:07.747353+0000 mgr.y (mgr.34107) 1210 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:09.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:08 smithi145 bash[79878]: cluster 2024-03-07T03:52:07.747353+0000 mgr.y (mgr.34107) 1210 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:09.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:08 smithi145 bash[82329]: cluster 2024-03-07T03:52:07.747353+0000 mgr.y (mgr.34107) 1210 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:10.468 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:52:11.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:10 smithi189 bash[66969]: cluster 2024-03-07T03:52:09.748203+0000 mgr.y (mgr.34107) 1211 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:11.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:10 smithi145 bash[79878]: cluster 2024-03-07T03:52:09.748203+0000 mgr.y (mgr.34107) 1211 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:11.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:10 smithi145 bash[82329]: cluster 2024-03-07T03:52:09.748203+0000 mgr.y (mgr.34107) 1211 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:12.591 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:52:12 smithi189 bash[59814]: ts=2024-03-07T03:52:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:52:12.819 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:52:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:52:12] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2024-03-07T03:52:13.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:12 smithi145 bash[79878]: cluster 2024-03-07T03:52:11.748886+0000 mgr.y (mgr.34107) 1212 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:13.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:12 smithi145 bash[82329]: cluster 2024-03-07T03:52:11.748886+0000 mgr.y (mgr.34107) 1212 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:13.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:12 smithi189 bash[66969]: cluster 2024-03-07T03:52:11.748886+0000 mgr.y (mgr.34107) 1212 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:13.933 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:52:13.933 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:52:15.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:14 smithi145 bash[82329]: cluster 2024-03-07T03:52:13.749641+0000 mgr.y (mgr.34107) 1213 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:15.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:14 smithi145 bash[82329]: audit 2024-03-07T03:52:13.935248+0000 mon.a (mon.0) 2002 : audit 0 from='client.? 172.21.15.145:0/3539229372' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:15.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:14 smithi145 bash[79878]: cluster 2024-03-07T03:52:13.749641+0000 mgr.y (mgr.34107) 1213 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:15.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:14 smithi145 bash[79878]: audit 2024-03-07T03:52:13.935248+0000 mon.a (mon.0) 2002 : audit 0 from='client.? 172.21.15.145:0/3539229372' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:15.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:14 smithi189 bash[66969]: cluster 2024-03-07T03:52:13.749641+0000 mgr.y (mgr.34107) 1213 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:15.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:14 smithi189 bash[66969]: audit 2024-03-07T03:52:13.935248+0000 mon.a (mon.0) 2002 : audit 0 from='client.? 172.21.15.145:0/3539229372' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:17.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:16 smithi145 bash[79878]: cluster 2024-03-07T03:52:15.750496+0000 mgr.y (mgr.34107) 1214 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:17.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:16 smithi145 bash[82329]: cluster 2024-03-07T03:52:15.750496+0000 mgr.y (mgr.34107) 1214 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:17.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:16 smithi189 bash[66969]: cluster 2024-03-07T03:52:15.750496+0000 mgr.y (mgr.34107) 1214 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:17.793 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:52:18.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:17 smithi145 bash[82329]: audit 2024-03-07T03:52:17.688622+0000 mon.a (mon.0) 2003 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:18.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:17 smithi145 bash[79878]: audit 2024-03-07T03:52:17.688622+0000 mon.a (mon.0) 2003 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:18.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:17 smithi189 bash[66969]: audit 2024-03-07T03:52:17.688622+0000 mon.a (mon.0) 2003 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:19.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:18 smithi145 bash[79878]: cluster 2024-03-07T03:52:17.751267+0000 mgr.y (mgr.34107) 1215 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:19.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:18 smithi145 bash[82329]: cluster 2024-03-07T03:52:17.751267+0000 mgr.y (mgr.34107) 1215 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:19.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:18 smithi189 bash[66969]: cluster 2024-03-07T03:52:17.751267+0000 mgr.y (mgr.34107) 1215 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:21.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:20 smithi145 bash[79878]: cluster 2024-03-07T03:52:19.752145+0000 mgr.y (mgr.34107) 1216 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:21.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:20 smithi145 bash[82329]: cluster 2024-03-07T03:52:19.752145+0000 mgr.y (mgr.34107) 1216 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:21.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:20 smithi189 bash[66969]: cluster 2024-03-07T03:52:19.752145+0000 mgr.y (mgr.34107) 1216 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:22.531 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:52:22.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:52:22 smithi189 bash[59814]: ts=2024-03-07T03:52:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:52:22.819 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:52:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:52:22] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2024-03-07T03:52:23.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:22 smithi145 bash[79878]: cluster 2024-03-07T03:52:21.753137+0000 mgr.y (mgr.34107) 1217 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:23.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:22 smithi145 bash[82329]: cluster 2024-03-07T03:52:21.753137+0000 mgr.y (mgr.34107) 1217 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:23.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:22 smithi189 bash[66969]: cluster 2024-03-07T03:52:21.753137+0000 mgr.y (mgr.34107) 1217 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:25.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:24 smithi145 bash[79878]: cluster 2024-03-07T03:52:23.754575+0000 mgr.y (mgr.34107) 1218 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:25.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:24 smithi145 bash[82329]: cluster 2024-03-07T03:52:23.754575+0000 mgr.y (mgr.34107) 1218 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:25.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:24 smithi189 bash[66969]: cluster 2024-03-07T03:52:23.754575+0000 mgr.y (mgr.34107) 1218 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:25.964 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:52:25.964 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:52:27.069 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:26 smithi145 bash[82329]: cluster 2024-03-07T03:52:25.755079+0000 mgr.y (mgr.34107) 1219 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:27.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:26 smithi145 bash[82329]: audit 2024-03-07T03:52:25.965508+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.145:0/1825315931' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:27.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:26 smithi145 bash[79878]: cluster 2024-03-07T03:52:25.755079+0000 mgr.y (mgr.34107) 1219 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:27.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:26 smithi145 bash[79878]: audit 2024-03-07T03:52:25.965508+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.145:0/1825315931' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:27.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:26 smithi189 bash[66969]: cluster 2024-03-07T03:52:25.755079+0000 mgr.y (mgr.34107) 1219 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:27.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:26 smithi189 bash[66969]: audit 2024-03-07T03:52:25.965508+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.145:0/1825315931' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:29.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:28 smithi145 bash[79878]: cluster 2024-03-07T03:52:27.755958+0000 mgr.y (mgr.34107) 1220 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:29.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:28 smithi145 bash[82329]: cluster 2024-03-07T03:52:27.755958+0000 mgr.y (mgr.34107) 1220 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:29.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:28 smithi189 bash[66969]: cluster 2024-03-07T03:52:27.755958+0000 mgr.y (mgr.34107) 1220 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:29.773 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:52:31.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:30 smithi145 bash[79878]: cluster 2024-03-07T03:52:29.756933+0000 mgr.y (mgr.34107) 1221 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:31.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:30 smithi145 bash[82329]: cluster 2024-03-07T03:52:29.756933+0000 mgr.y (mgr.34107) 1221 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:31.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:30 smithi189 bash[66969]: cluster 2024-03-07T03:52:29.756933+0000 mgr.y (mgr.34107) 1221 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:32.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:52:32 smithi189 bash[59814]: ts=2024-03-07T03:52:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:52:33.069 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:52:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:52:32] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2024-03-07T03:52:33.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:32 smithi145 bash[79878]: cluster 2024-03-07T03:52:31.757937+0000 mgr.y (mgr.34107) 1222 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:33.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:32 smithi145 bash[79878]: audit 2024-03-07T03:52:32.689150+0000 mon.a (mon.0) 2004 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:33.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:32 smithi145 bash[82329]: cluster 2024-03-07T03:52:31.757937+0000 mgr.y (mgr.34107) 1222 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:33.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:32 smithi145 bash[82329]: audit 2024-03-07T03:52:32.689150+0000 mon.a (mon.0) 2004 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:33.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:32 smithi189 bash[66969]: cluster 2024-03-07T03:52:31.757937+0000 mgr.y (mgr.34107) 1222 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:33.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:32 smithi189 bash[66969]: audit 2024-03-07T03:52:32.689150+0000 mon.a (mon.0) 2004 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:34.516 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:52:35.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:34 smithi145 bash[79878]: cluster 2024-03-07T03:52:33.758765+0000 mgr.y (mgr.34107) 1223 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:35.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:34 smithi145 bash[82329]: cluster 2024-03-07T03:52:33.758765+0000 mgr.y (mgr.34107) 1223 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:35.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:34 smithi189 bash[66969]: cluster 2024-03-07T03:52:33.758765+0000 mgr.y (mgr.34107) 1223 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:37.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:36 smithi145 bash[79878]: cluster 2024-03-07T03:52:35.759583+0000 mgr.y (mgr.34107) 1224 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:37.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:36 smithi145 bash[82329]: cluster 2024-03-07T03:52:35.759583+0000 mgr.y (mgr.34107) 1224 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:37.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:36 smithi189 bash[66969]: cluster 2024-03-07T03:52:35.759583+0000 mgr.y (mgr.34107) 1224 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:38.035 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:52:38.035 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:52:39.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:38 smithi145 bash[79878]: cluster 2024-03-07T03:52:37.760568+0000 mgr.y (mgr.34107) 1225 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:39.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:38 smithi145 bash[79878]: audit 2024-03-07T03:52:38.036684+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.145:0/3659606031' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:39.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:38 smithi145 bash[82329]: cluster 2024-03-07T03:52:37.760568+0000 mgr.y (mgr.34107) 1225 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:39.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:38 smithi145 bash[82329]: audit 2024-03-07T03:52:38.036684+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.145:0/3659606031' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:39.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:38 smithi189 bash[66969]: cluster 2024-03-07T03:52:37.760568+0000 mgr.y (mgr.34107) 1225 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:39.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:38 smithi189 bash[66969]: audit 2024-03-07T03:52:38.036684+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.145:0/3659606031' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:41.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:40 smithi145 bash[79878]: cluster 2024-03-07T03:52:39.761560+0000 mgr.y (mgr.34107) 1226 : cluster 0 pgmap v1950: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:41.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:40 smithi145 bash[82329]: cluster 2024-03-07T03:52:39.761560+0000 mgr.y (mgr.34107) 1226 : cluster 0 pgmap v1950: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:41.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:40 smithi189 bash[66969]: cluster 2024-03-07T03:52:39.761560+0000 mgr.y (mgr.34107) 1226 : cluster 0 pgmap v1950: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:41.844 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:52:42.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:52:42 smithi189 bash[59814]: ts=2024-03-07T03:52:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:52:43.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:42 smithi145 bash[79878]: cluster 2024-03-07T03:52:41.762697+0000 mgr.y (mgr.34107) 1227 : cluster 0 pgmap v1951: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:43.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:52:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:52:42] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2024-03-07T03:52:43.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:42 smithi145 bash[82329]: cluster 2024-03-07T03:52:41.762697+0000 mgr.y (mgr.34107) 1227 : cluster 0 pgmap v1951: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:43.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:42 smithi189 bash[66969]: cluster 2024-03-07T03:52:41.762697+0000 mgr.y (mgr.34107) 1227 : cluster 0 pgmap v1951: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:45.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:44 smithi145 bash[79878]: cluster 2024-03-07T03:52:43.763521+0000 mgr.y (mgr.34107) 1228 : cluster 0 pgmap v1952: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:45.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:44 smithi145 bash[82329]: cluster 2024-03-07T03:52:43.763521+0000 mgr.y (mgr.34107) 1228 : cluster 0 pgmap v1952: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:45.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:44 smithi189 bash[66969]: cluster 2024-03-07T03:52:43.763521+0000 mgr.y (mgr.34107) 1228 : cluster 0 pgmap v1952: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:46.582 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:52:47.322 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:46 smithi145 bash[79878]: cluster 2024-03-07T03:52:45.764314+0000 mgr.y (mgr.34107) 1229 : cluster 0 pgmap v1953: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:47.322 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:46 smithi145 bash[82329]: cluster 2024-03-07T03:52:45.764314+0000 mgr.y (mgr.34107) 1229 : cluster 0 pgmap v1953: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:47.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:46 smithi189 bash[66969]: cluster 2024-03-07T03:52:45.764314+0000 mgr.y (mgr.34107) 1229 : cluster 0 pgmap v1953: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:48.198 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:47 smithi145 bash[79878]: audit 2024-03-07T03:52:47.689534+0000 mon.a (mon.0) 2005 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:48.198 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:47 smithi145 bash[82329]: audit 2024-03-07T03:52:47.689534+0000 mon.a (mon.0) 2005 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:48.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:47 smithi189 bash[66969]: audit 2024-03-07T03:52:47.689534+0000 mon.a (mon.0) 2005 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:49.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:48 smithi145 bash[79878]: cluster 2024-03-07T03:52:47.765083+0000 mgr.y (mgr.34107) 1230 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:49.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:48 smithi145 bash[82329]: cluster 2024-03-07T03:52:47.765083+0000 mgr.y (mgr.34107) 1230 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:49.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:48 smithi189 bash[66969]: cluster 2024-03-07T03:52:47.765083+0000 mgr.y (mgr.34107) 1230 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:50.003 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:52:50.003 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:52:50.231 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:49 smithi145 bash[79878]: audit 2024-03-07T03:52:49.818762+0000 mon.a (mon.0) 2006 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:52:50.231 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:49 smithi145 bash[82329]: audit 2024-03-07T03:52:49.818762+0000 mon.a (mon.0) 2006 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:52:50.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:49 smithi189 bash[66969]: audit 2024-03-07T03:52:49.818762+0000 mon.a (mon.0) 2006 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:52:51.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:50 smithi145 bash[79878]: cluster 2024-03-07T03:52:49.765576+0000 mgr.y (mgr.34107) 1231 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:51.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:50 smithi145 bash[79878]: audit 2024-03-07T03:52:50.005374+0000 mon.a (mon.0) 2007 : audit 0 from='client.? 172.21.15.145:0/3847698366' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:51.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:50 smithi145 bash[82329]: cluster 2024-03-07T03:52:49.765576+0000 mgr.y (mgr.34107) 1231 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:51.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:50 smithi145 bash[82329]: audit 2024-03-07T03:52:50.005374+0000 mon.a (mon.0) 2007 : audit 0 from='client.? 172.21.15.145:0/3847698366' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:51.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:50 smithi189 bash[66969]: cluster 2024-03-07T03:52:49.765576+0000 mgr.y (mgr.34107) 1231 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:51.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:50 smithi189 bash[66969]: audit 2024-03-07T03:52:50.005374+0000 mon.a (mon.0) 2007 : audit 0 from='client.? 172.21.15.145:0/3847698366' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:52.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:52:52 smithi189 bash[59814]: ts=2024-03-07T03:52:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:52:53.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:52 smithi145 bash[79878]: cluster 2024-03-07T03:52:51.766543+0000 mgr.y (mgr.34107) 1232 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:53.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:52:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:52:52] "GET /metrics HTTP/1.1" 200 34153 "" "Prometheus/2.43.0" 2024-03-07T03:52:53.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:52 smithi145 bash[82329]: cluster 2024-03-07T03:52:51.766543+0000 mgr.y (mgr.34107) 1232 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:53.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:52 smithi189 bash[66969]: cluster 2024-03-07T03:52:51.766543+0000 mgr.y (mgr.34107) 1232 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:53.782 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:52:55.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:54 smithi145 bash[79878]: cluster 2024-03-07T03:52:53.767341+0000 mgr.y (mgr.34107) 1233 : cluster 0 pgmap v1957: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:55.319 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:54 smithi145 bash[82329]: cluster 2024-03-07T03:52:53.767341+0000 mgr.y (mgr.34107) 1233 : cluster 0 pgmap v1957: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:55.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:54 smithi189 bash[66969]: cluster 2024-03-07T03:52:53.767341+0000 mgr.y (mgr.34107) 1233 : cluster 0 pgmap v1957: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:57.069 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:56 smithi145 bash[79878]: audit 2024-03-07T03:52:55.660949+0000 mon.a (mon.0) 2008 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:56 smithi145 bash[79878]: audit 2024-03-07T03:52:55.670130+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:56 smithi145 bash[79878]: cluster 2024-03-07T03:52:55.767883+0000 mgr.y (mgr.34107) 1234 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:57.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:56 smithi145 bash[79878]: audit 2024-03-07T03:52:55.917242+0000 mon.a (mon.0) 2010 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:56 smithi145 bash[79878]: audit 2024-03-07T03:52:55.922392+0000 mon.a (mon.0) 2011 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:56 smithi145 bash[82329]: audit 2024-03-07T03:52:55.660949+0000 mon.a (mon.0) 2008 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:56 smithi145 bash[82329]: audit 2024-03-07T03:52:55.670130+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:56 smithi145 bash[82329]: cluster 2024-03-07T03:52:55.767883+0000 mgr.y (mgr.34107) 1234 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:57.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:56 smithi145 bash[82329]: audit 2024-03-07T03:52:55.917242+0000 mon.a (mon.0) 2010 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:56 smithi145 bash[82329]: audit 2024-03-07T03:52:55.922392+0000 mon.a (mon.0) 2011 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:56 smithi189 bash[66969]: audit 2024-03-07T03:52:55.660949+0000 mon.a (mon.0) 2008 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:56 smithi189 bash[66969]: audit 2024-03-07T03:52:55.670130+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:56 smithi189 bash[66969]: cluster 2024-03-07T03:52:55.767883+0000 mgr.y (mgr.34107) 1234 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:57.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:56 smithi189 bash[66969]: audit 2024-03-07T03:52:55.917242+0000 mon.a (mon.0) 2010 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:56 smithi189 bash[66969]: audit 2024-03-07T03:52:55.922392+0000 mon.a (mon.0) 2011 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.928 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:52:57.948 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:57 smithi145 bash[79878]: audit 2024-03-07T03:52:56.678852+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.948 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:57 smithi145 bash[79878]: audit 2024-03-07T03:52:56.686034+0000 mon.a (mon.0) 2013 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.948 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:57 smithi145 bash[79878]: audit 2024-03-07T03:52:56.933058+0000 mon.a (mon.0) 2014 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.948 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:57 smithi145 bash[79878]: audit 2024-03-07T03:52:56.943491+0000 mon.a (mon.0) 2015 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.948 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:57 smithi145 bash[82329]: audit 2024-03-07T03:52:56.678852+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.949 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:57 smithi145 bash[82329]: audit 2024-03-07T03:52:56.686034+0000 mon.a (mon.0) 2013 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.949 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:57 smithi145 bash[82329]: audit 2024-03-07T03:52:56.933058+0000 mon.a (mon.0) 2014 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:57.949 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:57 smithi145 bash[82329]: audit 2024-03-07T03:52:56.943491+0000 mon.a (mon.0) 2015 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:58.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:57 smithi189 bash[66969]: audit 2024-03-07T03:52:56.678852+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:58.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:57 smithi189 bash[66969]: audit 2024-03-07T03:52:56.686034+0000 mon.a (mon.0) 2013 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:58.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:57 smithi189 bash[66969]: audit 2024-03-07T03:52:56.933058+0000 mon.a (mon.0) 2014 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:58.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:57 smithi189 bash[66969]: audit 2024-03-07T03:52:56.943491+0000 mon.a (mon.0) 2015 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:52:59.053 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:52:58 smithi145 bash[82329]: cluster 2024-03-07T03:52:57.768649+0000 mgr.y (mgr.34107) 1235 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:59.053 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:52:58 smithi145 bash[79878]: cluster 2024-03-07T03:52:57.768649+0000 mgr.y (mgr.34107) 1235 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:59.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:52:58 smithi189 bash[66969]: cluster 2024-03-07T03:52:57.768649+0000 mgr.y (mgr.34107) 1235 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:01.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:00 smithi189 bash[66969]: cluster 2024-03-07T03:52:59.769559+0000 mgr.y (mgr.34107) 1236 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:01.155 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:00 smithi145 bash[79878]: cluster 2024-03-07T03:52:59.769559+0000 mgr.y (mgr.34107) 1236 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:01.155 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:00 smithi145 bash[82329]: cluster 2024-03-07T03:52:59.769559+0000 mgr.y (mgr.34107) 1236 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:01.337 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:53:01.337 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:53:02.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:01 smithi189 bash[66969]: audit 2024-03-07T03:53:01.338905+0000 mon.a (mon.0) 2016 : audit 0 from='client.? 172.21.15.145:0/2616595905' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:02.098 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:01 smithi145 bash[79878]: audit 2024-03-07T03:53:01.338905+0000 mon.a (mon.0) 2016 : audit 0 from='client.? 172.21.15.145:0/2616595905' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:02.098 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:01 smithi145 bash[82329]: audit 2024-03-07T03:53:01.338905+0000 mon.a (mon.0) 2016 : audit 0 from='client.? 172.21.15.145:0/2616595905' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:02.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:53:02 smithi189 bash[59814]: ts=2024-03-07T03:53:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:53:03.025 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:02 smithi145 bash[79878]: cluster 2024-03-07T03:53:01.770517+0000 mgr.y (mgr.34107) 1237 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:03.025 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:02 smithi145 bash[79878]: audit 2024-03-07T03:53:02.689967+0000 mon.a (mon.0) 2017 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:03.026 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:53:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:53:02] "GET /metrics HTTP/1.1" 200 34154 "" "Prometheus/2.43.0" 2024-03-07T03:53:03.026 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:02 smithi145 bash[82329]: cluster 2024-03-07T03:53:01.770517+0000 mgr.y (mgr.34107) 1237 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:03.026 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:02 smithi145 bash[82329]: audit 2024-03-07T03:53:02.689967+0000 mon.a (mon.0) 2017 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:03.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:02 smithi189 bash[66969]: cluster 2024-03-07T03:53:01.770517+0000 mgr.y (mgr.34107) 1237 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:03.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:02 smithi189 bash[66969]: audit 2024-03-07T03:53:02.689967+0000 mon.a (mon.0) 2017 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:05.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:04 smithi189 bash[66969]: cluster 2024-03-07T03:53:03.771325+0000 mgr.y (mgr.34107) 1238 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:05.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:04 smithi145 bash[79878]: cluster 2024-03-07T03:53:03.771325+0000 mgr.y (mgr.34107) 1238 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:05.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:04 smithi145 bash[82329]: cluster 2024-03-07T03:53:03.771325+0000 mgr.y (mgr.34107) 1238 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:05.897 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:53:07.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:06 smithi145 bash[79878]: cluster 2024-03-07T03:53:05.771951+0000 mgr.y (mgr.34107) 1239 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:07.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:06 smithi145 bash[79878]: audit 2024-03-07T03:53:06.700240+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:07.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:06 smithi145 bash[79878]: audit 2024-03-07T03:53:06.708872+0000 mon.a (mon.0) 2019 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:07.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:06 smithi145 bash[82329]: cluster 2024-03-07T03:53:05.771951+0000 mgr.y (mgr.34107) 1239 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:07.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:06 smithi145 bash[82329]: audit 2024-03-07T03:53:06.700240+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:07.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:06 smithi145 bash[82329]: audit 2024-03-07T03:53:06.708872+0000 mon.a (mon.0) 2019 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:07.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:06 smithi189 bash[66969]: cluster 2024-03-07T03:53:05.771951+0000 mgr.y (mgr.34107) 1239 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:07.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:06 smithi189 bash[66969]: audit 2024-03-07T03:53:06.700240+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:07.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:06 smithi189 bash[66969]: audit 2024-03-07T03:53:06.708872+0000 mon.a (mon.0) 2019 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:08.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:07 smithi145 bash[82329]: audit 2024-03-07T03:53:06.990939+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:08.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:07 smithi145 bash[82329]: audit 2024-03-07T03:53:07.001748+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:08.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:07 smithi145 bash[82329]: audit 2024-03-07T03:53:07.004044+0000 mon.a (mon.0) 2022 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:53:08.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:07 smithi145 bash[82329]: audit 2024-03-07T03:53:07.005588+0000 mon.a (mon.0) 2023 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:53:08.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:07 smithi145 bash[82329]: audit 2024-03-07T03:53:07.015432+0000 mon.a (mon.0) 2024 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:08.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:07 smithi145 bash[79878]: audit 2024-03-07T03:53:06.990939+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:08.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:07 smithi145 bash[79878]: audit 2024-03-07T03:53:07.001748+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:08.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:07 smithi145 bash[79878]: audit 2024-03-07T03:53:07.004044+0000 mon.a (mon.0) 2022 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:53:08.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:07 smithi145 bash[79878]: audit 2024-03-07T03:53:07.005588+0000 mon.a (mon.0) 2023 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:53:08.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:07 smithi145 bash[79878]: audit 2024-03-07T03:53:07.015432+0000 mon.a (mon.0) 2024 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:08.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:07 smithi189 bash[66969]: audit 2024-03-07T03:53:06.990939+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:08.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:07 smithi189 bash[66969]: audit 2024-03-07T03:53:07.001748+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:08.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:07 smithi189 bash[66969]: audit 2024-03-07T03:53:07.004044+0000 mon.a (mon.0) 2022 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:53:08.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:07 smithi189 bash[66969]: audit 2024-03-07T03:53:07.005588+0000 mon.a (mon.0) 2023 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:53:08.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:07 smithi189 bash[66969]: audit 2024-03-07T03:53:07.015432+0000 mon.a (mon.0) 2024 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:53:09.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:09 smithi145 bash[79878]: cluster 2024-03-07T03:53:07.772835+0000 mgr.y (mgr.34107) 1240 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:09.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:09 smithi145 bash[82329]: cluster 2024-03-07T03:53:07.772835+0000 mgr.y (mgr.34107) 1240 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:09.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:09 smithi189 bash[66969]: cluster 2024-03-07T03:53:07.772835+0000 mgr.y (mgr.34107) 1240 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:10.628 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:53:11.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:11 smithi145 bash[82329]: cluster 2024-03-07T03:53:09.773760+0000 mgr.y (mgr.34107) 1241 : cluster 0 pgmap v1965: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:11.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:11 smithi145 bash[79878]: cluster 2024-03-07T03:53:09.773760+0000 mgr.y (mgr.34107) 1241 : cluster 0 pgmap v1965: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:11.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:11 smithi189 bash[66969]: cluster 2024-03-07T03:53:09.773760+0000 mgr.y (mgr.34107) 1241 : cluster 0 pgmap v1965: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:12.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:53:12 smithi189 bash[59814]: ts=2024-03-07T03:53:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:53:13.021 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:53:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:53:12] "GET /metrics HTTP/1.1" 200 34154 "" "Prometheus/2.43.0" 2024-03-07T03:53:13.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:13 smithi145 bash[79878]: cluster 2024-03-07T03:53:11.774474+0000 mgr.y (mgr.34107) 1242 : cluster 0 pgmap v1966: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:13.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:13 smithi145 bash[82329]: cluster 2024-03-07T03:53:11.774474+0000 mgr.y (mgr.34107) 1242 : cluster 0 pgmap v1966: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:13.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:13 smithi189 bash[66969]: cluster 2024-03-07T03:53:11.774474+0000 mgr.y (mgr.34107) 1242 : cluster 0 pgmap v1966: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:13.805 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:53:13.806 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:53:14.319 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:14 smithi145 bash[79878]: audit 2024-03-07T03:53:13.807569+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.145:0/3669784746' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:14.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:14 smithi145 bash[82329]: audit 2024-03-07T03:53:13.807569+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.145:0/3669784746' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:14.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:14 smithi189 bash[66969]: audit 2024-03-07T03:53:13.807569+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.145:0/3669784746' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:15.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:15 smithi145 bash[79878]: cluster 2024-03-07T03:53:13.775212+0000 mgr.y (mgr.34107) 1243 : cluster 0 pgmap v1967: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:15.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:15 smithi145 bash[82329]: cluster 2024-03-07T03:53:13.775212+0000 mgr.y (mgr.34107) 1243 : cluster 0 pgmap v1967: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:15.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:15 smithi189 bash[66969]: cluster 2024-03-07T03:53:13.775212+0000 mgr.y (mgr.34107) 1243 : cluster 0 pgmap v1967: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:17.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:17 smithi145 bash[79878]: cluster 2024-03-07T03:53:15.776088+0000 mgr.y (mgr.34107) 1244 : cluster 0 pgmap v1968: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:17.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:17 smithi145 bash[82329]: cluster 2024-03-07T03:53:15.776088+0000 mgr.y (mgr.34107) 1244 : cluster 0 pgmap v1968: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:17.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:17 smithi189 bash[66969]: cluster 2024-03-07T03:53:15.776088+0000 mgr.y (mgr.34107) 1244 : cluster 0 pgmap v1968: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:17.655 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:53:18.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:18 smithi145 bash[79878]: audit 2024-03-07T03:53:17.690337+0000 mon.a (mon.0) 2025 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:18.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:18 smithi145 bash[82329]: audit 2024-03-07T03:53:17.690337+0000 mon.a (mon.0) 2025 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:18.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:18 smithi189 bash[66969]: audit 2024-03-07T03:53:17.690337+0000 mon.a (mon.0) 2025 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:19.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:19 smithi145 bash[79878]: cluster 2024-03-07T03:53:17.776904+0000 mgr.y (mgr.34107) 1245 : cluster 0 pgmap v1969: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:19.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:19 smithi145 bash[82329]: cluster 2024-03-07T03:53:17.776904+0000 mgr.y (mgr.34107) 1245 : cluster 0 pgmap v1969: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:19.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:19 smithi189 bash[66969]: cluster 2024-03-07T03:53:17.776904+0000 mgr.y (mgr.34107) 1245 : cluster 0 pgmap v1969: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:21.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:21 smithi145 bash[79878]: cluster 2024-03-07T03:53:19.777947+0000 mgr.y (mgr.34107) 1246 : cluster 0 pgmap v1970: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:21.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:21 smithi145 bash[82329]: cluster 2024-03-07T03:53:19.777947+0000 mgr.y (mgr.34107) 1246 : cluster 0 pgmap v1970: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:21.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:21 smithi189 bash[66969]: cluster 2024-03-07T03:53:19.777947+0000 mgr.y (mgr.34107) 1246 : cluster 0 pgmap v1970: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:22.396 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:53:22.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:53:22 smithi189 bash[59814]: ts=2024-03-07T03:53:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:53:23.056 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:53:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:53:22] "GET /metrics HTTP/1.1" 200 34154 "" "Prometheus/2.43.0" 2024-03-07T03:53:23.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:23 smithi145 bash[79878]: cluster 2024-03-07T03:53:21.778883+0000 mgr.y (mgr.34107) 1247 : cluster 0 pgmap v1971: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:23.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:23 smithi145 bash[82329]: cluster 2024-03-07T03:53:21.778883+0000 mgr.y (mgr.34107) 1247 : cluster 0 pgmap v1971: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:23.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:23 smithi189 bash[66969]: cluster 2024-03-07T03:53:21.778883+0000 mgr.y (mgr.34107) 1247 : cluster 0 pgmap v1971: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:25.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:25 smithi145 bash[79878]: cluster 2024-03-07T03:53:23.779634+0000 mgr.y (mgr.34107) 1248 : cluster 0 pgmap v1972: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:25.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:25 smithi145 bash[82329]: cluster 2024-03-07T03:53:23.779634+0000 mgr.y (mgr.34107) 1248 : cluster 0 pgmap v1972: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:25.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:25 smithi189 bash[66969]: cluster 2024-03-07T03:53:23.779634+0000 mgr.y (mgr.34107) 1248 : cluster 0 pgmap v1972: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:25.799 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:53:25.799 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:53:26.064 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:26 smithi145 bash[79878]: audit 2024-03-07T03:53:25.800387+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.145:0/2509489676' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:26.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:26 smithi145 bash[82329]: audit 2024-03-07T03:53:25.800387+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.145:0/2509489676' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:26.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:26 smithi189 bash[66969]: audit 2024-03-07T03:53:25.800387+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.145:0/2509489676' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:27.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:27 smithi189 bash[66969]: cluster 2024-03-07T03:53:25.780388+0000 mgr.y (mgr.34107) 1249 : cluster 0 pgmap v1973: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:27.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:27 smithi145 bash[79878]: cluster 2024-03-07T03:53:25.780388+0000 mgr.y (mgr.34107) 1249 : cluster 0 pgmap v1973: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:27.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:27 smithi145 bash[82329]: cluster 2024-03-07T03:53:25.780388+0000 mgr.y (mgr.34107) 1249 : cluster 0 pgmap v1973: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:29.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:29 smithi189 bash[66969]: cluster 2024-03-07T03:53:27.781212+0000 mgr.y (mgr.34107) 1250 : cluster 0 pgmap v1974: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:29.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:29 smithi145 bash[79878]: cluster 2024-03-07T03:53:27.781212+0000 mgr.y (mgr.34107) 1250 : cluster 0 pgmap v1974: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:29.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:29 smithi145 bash[82329]: cluster 2024-03-07T03:53:27.781212+0000 mgr.y (mgr.34107) 1250 : cluster 0 pgmap v1974: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:29.685 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:53:31.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:31 smithi189 bash[66969]: cluster 2024-03-07T03:53:29.781922+0000 mgr.y (mgr.34107) 1251 : cluster 0 pgmap v1975: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:31.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:31 smithi145 bash[79878]: cluster 2024-03-07T03:53:29.781922+0000 mgr.y (mgr.34107) 1251 : cluster 0 pgmap v1975: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:31.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:31 smithi145 bash[82329]: cluster 2024-03-07T03:53:29.781922+0000 mgr.y (mgr.34107) 1251 : cluster 0 pgmap v1975: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:32.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:53:32 smithi189 bash[59814]: ts=2024-03-07T03:53:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:53:33.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:53:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:53:32] "GET /metrics HTTP/1.1" 200 34154 "" "Prometheus/2.43.0" 2024-03-07T03:53:33.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:33 smithi189 bash[66969]: cluster 2024-03-07T03:53:31.782837+0000 mgr.y (mgr.34107) 1252 : cluster 0 pgmap v1976: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:33.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:33 smithi189 bash[66969]: audit 2024-03-07T03:53:32.690840+0000 mon.a (mon.0) 2026 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:33.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:33 smithi145 bash[82329]: cluster 2024-03-07T03:53:31.782837+0000 mgr.y (mgr.34107) 1252 : cluster 0 pgmap v1976: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:33.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:33 smithi145 bash[82329]: audit 2024-03-07T03:53:32.690840+0000 mon.a (mon.0) 2026 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:33.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:33 smithi145 bash[79878]: cluster 2024-03-07T03:53:31.782837+0000 mgr.y (mgr.34107) 1252 : cluster 0 pgmap v1976: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:33.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:33 smithi145 bash[79878]: audit 2024-03-07T03:53:32.690840+0000 mon.a (mon.0) 2026 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:34.427 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:53:35.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:35 smithi189 bash[66969]: cluster 2024-03-07T03:53:33.783619+0000 mgr.y (mgr.34107) 1253 : cluster 0 pgmap v1977: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:35.488 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:35 smithi145 bash[79878]: cluster 2024-03-07T03:53:33.783619+0000 mgr.y (mgr.34107) 1253 : cluster 0 pgmap v1977: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:35.489 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:35 smithi145 bash[82329]: cluster 2024-03-07T03:53:33.783619+0000 mgr.y (mgr.34107) 1253 : cluster 0 pgmap v1977: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:37.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:37 smithi145 bash[82329]: cluster 2024-03-07T03:53:35.784473+0000 mgr.y (mgr.34107) 1254 : cluster 0 pgmap v1978: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:37.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:37 smithi145 bash[79878]: cluster 2024-03-07T03:53:35.784473+0000 mgr.y (mgr.34107) 1254 : cluster 0 pgmap v1978: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:37.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:37 smithi189 bash[66969]: cluster 2024-03-07T03:53:35.784473+0000 mgr.y (mgr.34107) 1254 : cluster 0 pgmap v1978: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:37.700 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:53:37.700 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:53:38.467 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:38 smithi145 bash[79878]: audit 2024-03-07T03:53:37.702169+0000 mon.a (mon.0) 2027 : audit 0 from='client.? 172.21.15.145:0/1243812151' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:38.467 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:38 smithi145 bash[82329]: audit 2024-03-07T03:53:37.702169+0000 mon.a (mon.0) 2027 : audit 0 from='client.? 172.21.15.145:0/1243812151' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:38.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:38 smithi189 bash[66969]: audit 2024-03-07T03:53:37.702169+0000 mon.a (mon.0) 2027 : audit 0 from='client.? 172.21.15.145:0/1243812151' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:39.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:39 smithi145 bash[79878]: cluster 2024-03-07T03:53:37.785004+0000 mgr.y (mgr.34107) 1255 : cluster 0 pgmap v1979: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:39.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:39 smithi145 bash[82329]: cluster 2024-03-07T03:53:37.785004+0000 mgr.y (mgr.34107) 1255 : cluster 0 pgmap v1979: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:39.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:39 smithi189 bash[66969]: cluster 2024-03-07T03:53:37.785004+0000 mgr.y (mgr.34107) 1255 : cluster 0 pgmap v1979: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:41.469 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:53:41.480 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:41 smithi145 bash[79878]: cluster 2024-03-07T03:53:39.785861+0000 mgr.y (mgr.34107) 1256 : cluster 0 pgmap v1980: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:41.480 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:41 smithi145 bash[82329]: cluster 2024-03-07T03:53:39.785861+0000 mgr.y (mgr.34107) 1256 : cluster 0 pgmap v1980: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:41.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:41 smithi189 bash[66969]: cluster 2024-03-07T03:53:39.785861+0000 mgr.y (mgr.34107) 1256 : cluster 0 pgmap v1980: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:42.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:53:42 smithi189 bash[59814]: ts=2024-03-07T03:53:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:53:43.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:53:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:53:42] "GET /metrics HTTP/1.1" 200 34154 "" "Prometheus/2.43.0" 2024-03-07T03:53:43.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:43 smithi145 bash[79878]: cluster 2024-03-07T03:53:41.786971+0000 mgr.y (mgr.34107) 1257 : cluster 0 pgmap v1981: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:43.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:43 smithi145 bash[82329]: cluster 2024-03-07T03:53:41.786971+0000 mgr.y (mgr.34107) 1257 : cluster 0 pgmap v1981: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:43.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:43 smithi189 bash[66969]: cluster 2024-03-07T03:53:41.786971+0000 mgr.y (mgr.34107) 1257 : cluster 0 pgmap v1981: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:45.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:45 smithi145 bash[79878]: cluster 2024-03-07T03:53:43.787838+0000 mgr.y (mgr.34107) 1258 : cluster 0 pgmap v1982: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:45.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:45 smithi145 bash[82329]: cluster 2024-03-07T03:53:43.787838+0000 mgr.y (mgr.34107) 1258 : cluster 0 pgmap v1982: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:45.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:45 smithi189 bash[66969]: cluster 2024-03-07T03:53:43.787838+0000 mgr.y (mgr.34107) 1258 : cluster 0 pgmap v1982: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:46.210 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:53:47.437 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:47 smithi145 bash[79878]: cluster 2024-03-07T03:53:45.788709+0000 mgr.y (mgr.34107) 1259 : cluster 0 pgmap v1983: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:47.438 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:47 smithi145 bash[82329]: cluster 2024-03-07T03:53:45.788709+0000 mgr.y (mgr.34107) 1259 : cluster 0 pgmap v1983: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:47.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:47 smithi189 bash[66969]: cluster 2024-03-07T03:53:45.788709+0000 mgr.y (mgr.34107) 1259 : cluster 0 pgmap v1983: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:48.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:48 smithi145 bash[79878]: audit 2024-03-07T03:53:47.691485+0000 mon.a (mon.0) 2028 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:48.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:48 smithi145 bash[82329]: audit 2024-03-07T03:53:47.691485+0000 mon.a (mon.0) 2028 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:48.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:48 smithi189 bash[66969]: audit 2024-03-07T03:53:47.691485+0000 mon.a (mon.0) 2028 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:49.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:49 smithi145 bash[79878]: cluster 2024-03-07T03:53:47.789508+0000 mgr.y (mgr.34107) 1260 : cluster 0 pgmap v1984: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:49.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:49 smithi145 bash[82329]: cluster 2024-03-07T03:53:47.789508+0000 mgr.y (mgr.34107) 1260 : cluster 0 pgmap v1984: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:49.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:49 smithi189 bash[66969]: cluster 2024-03-07T03:53:47.789508+0000 mgr.y (mgr.34107) 1260 : cluster 0 pgmap v1984: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:49.712 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:53:49.712 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:53:50.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:50 smithi145 bash[82329]: audit 2024-03-07T03:53:49.713636+0000 mon.a (mon.0) 2029 : audit 0 from='client.? 172.21.15.145:0/640682077' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:50.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:50 smithi145 bash[79878]: audit 2024-03-07T03:53:49.713636+0000 mon.a (mon.0) 2029 : audit 0 from='client.? 172.21.15.145:0/640682077' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:50.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:50 smithi189 bash[66969]: audit 2024-03-07T03:53:49.713636+0000 mon.a (mon.0) 2029 : audit 0 from='client.? 172.21.15.145:0/640682077' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:51.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:51 smithi145 bash[79878]: cluster 2024-03-07T03:53:49.789998+0000 mgr.y (mgr.34107) 1261 : cluster 0 pgmap v1985: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:51.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:51 smithi145 bash[82329]: cluster 2024-03-07T03:53:49.789998+0000 mgr.y (mgr.34107) 1261 : cluster 0 pgmap v1985: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:51.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:51 smithi189 bash[66969]: cluster 2024-03-07T03:53:49.789998+0000 mgr.y (mgr.34107) 1261 : cluster 0 pgmap v1985: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:52.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:53:52 smithi189 bash[59814]: ts=2024-03-07T03:53:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:53:53.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:53:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:53:52] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2024-03-07T03:53:53.534 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:53:53.545 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:53 smithi145 bash[82329]: cluster 2024-03-07T03:53:51.790893+0000 mgr.y (mgr.34107) 1262 : cluster 0 pgmap v1986: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:53.546 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:53 smithi145 bash[79878]: cluster 2024-03-07T03:53:51.790893+0000 mgr.y (mgr.34107) 1262 : cluster 0 pgmap v1986: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:53.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:53 smithi189 bash[66969]: cluster 2024-03-07T03:53:51.790893+0000 mgr.y (mgr.34107) 1262 : cluster 0 pgmap v1986: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:55.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:55 smithi145 bash[79878]: cluster 2024-03-07T03:53:53.791779+0000 mgr.y (mgr.34107) 1263 : cluster 0 pgmap v1987: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:55.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:55 smithi145 bash[82329]: cluster 2024-03-07T03:53:53.791779+0000 mgr.y (mgr.34107) 1263 : cluster 0 pgmap v1987: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:55.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:55 smithi189 bash[66969]: cluster 2024-03-07T03:53:53.791779+0000 mgr.y (mgr.34107) 1263 : cluster 0 pgmap v1987: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:57.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:57 smithi145 bash[79878]: cluster 2024-03-07T03:53:55.792582+0000 mgr.y (mgr.34107) 1264 : cluster 0 pgmap v1988: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:57.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:57 smithi145 bash[82329]: cluster 2024-03-07T03:53:55.792582+0000 mgr.y (mgr.34107) 1264 : cluster 0 pgmap v1988: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:57.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:57 smithi189 bash[66969]: cluster 2024-03-07T03:53:55.792582+0000 mgr.y (mgr.34107) 1264 : cluster 0 pgmap v1988: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:58.277 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:53:59.436 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:53:59 smithi145 bash[79878]: cluster 2024-03-07T03:53:57.793373+0000 mgr.y (mgr.34107) 1265 : cluster 0 pgmap v1989: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:59.437 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:53:59 smithi145 bash[82329]: cluster 2024-03-07T03:53:57.793373+0000 mgr.y (mgr.34107) 1265 : cluster 0 pgmap v1989: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:59.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:53:59 smithi189 bash[66969]: cluster 2024-03-07T03:53:57.793373+0000 mgr.y (mgr.34107) 1265 : cluster 0 pgmap v1989: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:01.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:01 smithi145 bash[79878]: cluster 2024-03-07T03:53:59.793892+0000 mgr.y (mgr.34107) 1266 : cluster 0 pgmap v1990: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:01.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:01 smithi145 bash[82329]: cluster 2024-03-07T03:53:59.793892+0000 mgr.y (mgr.34107) 1266 : cluster 0 pgmap v1990: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:01.450 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:54:01.450 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:54:01.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:01 smithi189 bash[66969]: cluster 2024-03-07T03:53:59.793892+0000 mgr.y (mgr.34107) 1266 : cluster 0 pgmap v1990: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:02.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:02 smithi145 bash[79878]: audit 2024-03-07T03:54:01.451946+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.145:0/2047572233' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:02.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:02 smithi145 bash[82329]: audit 2024-03-07T03:54:01.451946+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.145:0/2047572233' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:02.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:02 smithi189 bash[66969]: audit 2024-03-07T03:54:01.451946+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.145:0/2047572233' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:02.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:54:02 smithi189 bash[59814]: ts=2024-03-07T03:54:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:54:03.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:54:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:54:02] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2024-03-07T03:54:03.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:03 smithi145 bash[79878]: cluster 2024-03-07T03:54:01.794936+0000 mgr.y (mgr.34107) 1267 : cluster 0 pgmap v1991: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:03.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:03 smithi145 bash[79878]: audit 2024-03-07T03:54:02.691723+0000 mon.a (mon.0) 2030 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:03.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:03 smithi145 bash[82329]: cluster 2024-03-07T03:54:01.794936+0000 mgr.y (mgr.34107) 1267 : cluster 0 pgmap v1991: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:03.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:03 smithi145 bash[82329]: audit 2024-03-07T03:54:02.691723+0000 mon.a (mon.0) 2030 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:03.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:03 smithi189 bash[66969]: cluster 2024-03-07T03:54:01.794936+0000 mgr.y (mgr.34107) 1267 : cluster 0 pgmap v1991: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:03.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:03 smithi189 bash[66969]: audit 2024-03-07T03:54:02.691723+0000 mon.a (mon.0) 2030 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:05.294 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:54:05.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:05 smithi145 bash[79878]: cluster 2024-03-07T03:54:03.795783+0000 mgr.y (mgr.34107) 1268 : cluster 0 pgmap v1992: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:05.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:05 smithi145 bash[82329]: cluster 2024-03-07T03:54:03.795783+0000 mgr.y (mgr.34107) 1268 : cluster 0 pgmap v1992: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:05.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:05 smithi189 bash[66969]: cluster 2024-03-07T03:54:03.795783+0000 mgr.y (mgr.34107) 1268 : cluster 0 pgmap v1992: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:07.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:07 smithi145 bash[79878]: cluster 2024-03-07T03:54:05.796666+0000 mgr.y (mgr.34107) 1269 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:07.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:07 smithi145 bash[79878]: audit 2024-03-07T03:54:07.098165+0000 mon.a (mon.0) 2031 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:54:07.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:07 smithi145 bash[82329]: cluster 2024-03-07T03:54:05.796666+0000 mgr.y (mgr.34107) 1269 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:07.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:07 smithi145 bash[82329]: audit 2024-03-07T03:54:07.098165+0000 mon.a (mon.0) 2031 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:54:07.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:07 smithi189 bash[66969]: cluster 2024-03-07T03:54:05.796666+0000 mgr.y (mgr.34107) 1269 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:07.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:07 smithi189 bash[66969]: audit 2024-03-07T03:54:07.098165+0000 mon.a (mon.0) 2031 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:54:09.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:09 smithi145 bash[79878]: cluster 2024-03-07T03:54:07.797531+0000 mgr.y (mgr.34107) 1270 : cluster 0 pgmap v1994: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:09.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:09 smithi145 bash[82329]: cluster 2024-03-07T03:54:07.797531+0000 mgr.y (mgr.34107) 1270 : cluster 0 pgmap v1994: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:09.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:09 smithi189 bash[66969]: cluster 2024-03-07T03:54:07.797531+0000 mgr.y (mgr.34107) 1270 : cluster 0 pgmap v1994: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:10.037 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:54:11.458 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:11 smithi145 bash[79878]: cluster 2024-03-07T03:54:09.798410+0000 mgr.y (mgr.34107) 1271 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:11.458 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:11 smithi145 bash[82329]: cluster 2024-03-07T03:54:09.798410+0000 mgr.y (mgr.34107) 1271 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:11.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:11 smithi189 bash[66969]: cluster 2024-03-07T03:54:09.798410+0000 mgr.y (mgr.34107) 1271 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:12.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:54:12 smithi189 bash[59814]: ts=2024-03-07T03:54:12.313Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:54:13.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:54:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:54:12] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2024-03-07T03:54:13.284 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:54:13.284 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:54:13.540 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:13 smithi145 bash[82329]: cluster 2024-03-07T03:54:11.799464+0000 mgr.y (mgr.34107) 1272 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:13.540 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:13 smithi145 bash[82329]: audit 2024-03-07T03:54:13.145101+0000 mon.a (mon.0) 2032 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:13.540 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:13 smithi145 bash[82329]: audit 2024-03-07T03:54:13.150558+0000 mon.a (mon.0) 2033 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:13.540 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:13 smithi145 bash[82329]: audit 2024-03-07T03:54:13.155650+0000 mon.a (mon.0) 2034 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:13.540 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:13 smithi145 bash[82329]: audit 2024-03-07T03:54:13.160877+0000 mon.a (mon.0) 2035 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:13.541 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:13 smithi145 bash[79878]: cluster 2024-03-07T03:54:11.799464+0000 mgr.y (mgr.34107) 1272 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:13.541 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:13 smithi145 bash[79878]: audit 2024-03-07T03:54:13.145101+0000 mon.a (mon.0) 2032 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:13.541 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:13 smithi145 bash[79878]: audit 2024-03-07T03:54:13.150558+0000 mon.a (mon.0) 2033 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:13.541 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:13 smithi145 bash[79878]: audit 2024-03-07T03:54:13.155650+0000 mon.a (mon.0) 2034 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:13.541 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:13 smithi145 bash[79878]: audit 2024-03-07T03:54:13.160877+0000 mon.a (mon.0) 2035 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:13.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:13 smithi189 bash[66969]: cluster 2024-03-07T03:54:11.799464+0000 mgr.y (mgr.34107) 1272 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:13.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:13 smithi189 bash[66969]: audit 2024-03-07T03:54:13.145101+0000 mon.a (mon.0) 2032 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:13.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:13 smithi189 bash[66969]: audit 2024-03-07T03:54:13.150558+0000 mon.a (mon.0) 2033 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:13.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:13 smithi189 bash[66969]: audit 2024-03-07T03:54:13.155650+0000 mon.a (mon.0) 2034 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:13.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:13 smithi189 bash[66969]: audit 2024-03-07T03:54:13.160877+0000 mon.a (mon.0) 2035 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:14.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:14 smithi145 bash[79878]: audit 2024-03-07T03:54:13.285338+0000 mon.a (mon.0) 2036 : audit 0 from='client.? 172.21.15.145:0/1346587384' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:14.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:14 smithi145 bash[79878]: audit 2024-03-07T03:54:13.790137+0000 mon.a (mon.0) 2037 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:54:14.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:14 smithi145 bash[79878]: audit 2024-03-07T03:54:13.792102+0000 mon.a (mon.0) 2038 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:54:14.571 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:14 smithi145 bash[79878]: audit 2024-03-07T03:54:13.801831+0000 mon.a (mon.0) 2039 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:14.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:14 smithi145 bash[82329]: audit 2024-03-07T03:54:13.285338+0000 mon.a (mon.0) 2036 : audit 0 from='client.? 172.21.15.145:0/1346587384' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:14.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:14 smithi145 bash[82329]: audit 2024-03-07T03:54:13.790137+0000 mon.a (mon.0) 2037 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:54:14.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:14 smithi145 bash[82329]: audit 2024-03-07T03:54:13.792102+0000 mon.a (mon.0) 2038 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:54:14.571 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:14 smithi145 bash[82329]: audit 2024-03-07T03:54:13.801831+0000 mon.a (mon.0) 2039 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:14.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:14 smithi189 bash[66969]: audit 2024-03-07T03:54:13.285338+0000 mon.a (mon.0) 2036 : audit 0 from='client.? 172.21.15.145:0/1346587384' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:14.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:14 smithi189 bash[66969]: audit 2024-03-07T03:54:13.790137+0000 mon.a (mon.0) 2037 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:54:14.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:14 smithi189 bash[66969]: audit 2024-03-07T03:54:13.792102+0000 mon.a (mon.0) 2038 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:54:14.591 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:14 smithi189 bash[66969]: audit 2024-03-07T03:54:13.801831+0000 mon.a (mon.0) 2039 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:54:15.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:15 smithi145 bash[79878]: cluster 2024-03-07T03:54:13.800048+0000 mgr.y (mgr.34107) 1273 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:15.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:15 smithi145 bash[82329]: cluster 2024-03-07T03:54:13.800048+0000 mgr.y (mgr.34107) 1273 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:15.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:15 smithi189 bash[66969]: cluster 2024-03-07T03:54:13.800048+0000 mgr.y (mgr.34107) 1273 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:16.570 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:16 smithi145 bash[79878]: cluster 2024-03-07T03:54:15.800981+0000 mgr.y (mgr.34107) 1274 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:16.570 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:16 smithi145 bash[82329]: cluster 2024-03-07T03:54:15.800981+0000 mgr.y (mgr.34107) 1274 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:16.590 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:16 smithi189 bash[66969]: cluster 2024-03-07T03:54:15.800981+0000 mgr.y (mgr.34107) 1274 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:17.140 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:54:18.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:17 smithi145 bash[82329]: audit 2024-03-07T03:54:17.692360+0000 mon.a (mon.0) 2040 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:18.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:17 smithi145 bash[79878]: audit 2024-03-07T03:54:17.692360+0000 mon.a (mon.0) 2040 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:18.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:17 smithi189 bash[66969]: audit 2024-03-07T03:54:17.692360+0000 mon.a (mon.0) 2040 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:19.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:18 smithi145 bash[79878]: cluster 2024-03-07T03:54:17.801902+0000 mgr.y (mgr.34107) 1275 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:19.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:18 smithi145 bash[82329]: cluster 2024-03-07T03:54:17.801902+0000 mgr.y (mgr.34107) 1275 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:19.090 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:18 smithi189 bash[66969]: cluster 2024-03-07T03:54:17.801902+0000 mgr.y (mgr.34107) 1275 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:21.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:20 smithi145 bash[79878]: cluster 2024-03-07T03:54:19.802816+0000 mgr.y (mgr.34107) 1276 : cluster 0 pgmap v2000: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:21.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:20 smithi145 bash[82329]: cluster 2024-03-07T03:54:19.802816+0000 mgr.y (mgr.34107) 1276 : cluster 0 pgmap v2000: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:21.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:20 smithi189 bash[66969]: cluster 2024-03-07T03:54:19.802816+0000 mgr.y (mgr.34107) 1276 : cluster 0 pgmap v2000: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:21.876 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:54:22.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:54:22 smithi189 bash[59814]: ts=2024-03-07T03:54:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:54:23.018 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:22 smithi145 bash[79878]: cluster 2024-03-07T03:54:21.803793+0000 mgr.y (mgr.34107) 1277 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:23.019 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:54:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:54:22] "GET /metrics HTTP/1.1" 200 34154 "" "Prometheus/2.43.0" 2024-03-07T03:54:23.019 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:22 smithi145 bash[82329]: cluster 2024-03-07T03:54:21.803793+0000 mgr.y (mgr.34107) 1277 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:23.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:22 smithi189 bash[66969]: cluster 2024-03-07T03:54:21.803793+0000 mgr.y (mgr.34107) 1277 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:25.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:24 smithi145 bash[79878]: cluster 2024-03-07T03:54:23.804628+0000 mgr.y (mgr.34107) 1278 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:25.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:24 smithi145 bash[82329]: cluster 2024-03-07T03:54:23.804628+0000 mgr.y (mgr.34107) 1278 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:25.229 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:54:25.229 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:54:25.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:24 smithi189 bash[66969]: cluster 2024-03-07T03:54:23.804628+0000 mgr.y (mgr.34107) 1278 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:26.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:25 smithi145 bash[79878]: audit 2024-03-07T03:54:25.230650+0000 mon.a (mon.0) 2041 : audit 0 from='client.? 172.21.15.145:0/2601140890' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:26.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:25 smithi145 bash[82329]: audit 2024-03-07T03:54:25.230650+0000 mon.a (mon.0) 2041 : audit 0 from='client.? 172.21.15.145:0/2601140890' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:26.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:25 smithi189 bash[66969]: audit 2024-03-07T03:54:25.230650+0000 mon.a (mon.0) 2041 : audit 0 from='client.? 172.21.15.145:0/2601140890' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:27.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:26 smithi145 bash[79878]: cluster 2024-03-07T03:54:25.805473+0000 mgr.y (mgr.34107) 1279 : cluster 0 pgmap v2003: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:27.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:26 smithi145 bash[82329]: cluster 2024-03-07T03:54:25.805473+0000 mgr.y (mgr.34107) 1279 : cluster 0 pgmap v2003: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:27.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:26 smithi189 bash[66969]: cluster 2024-03-07T03:54:25.805473+0000 mgr.y (mgr.34107) 1279 : cluster 0 pgmap v2003: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:29.063 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:54:29.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:28 smithi145 bash[79878]: cluster 2024-03-07T03:54:27.806333+0000 mgr.y (mgr.34107) 1280 : cluster 0 pgmap v2004: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:29.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:28 smithi145 bash[82329]: cluster 2024-03-07T03:54:27.806333+0000 mgr.y (mgr.34107) 1280 : cluster 0 pgmap v2004: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:29.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:28 smithi189 bash[66969]: cluster 2024-03-07T03:54:27.806333+0000 mgr.y (mgr.34107) 1280 : cluster 0 pgmap v2004: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:31.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:30 smithi145 bash[79878]: cluster 2024-03-07T03:54:29.807229+0000 mgr.y (mgr.34107) 1281 : cluster 0 pgmap v2005: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:31.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:30 smithi145 bash[82329]: cluster 2024-03-07T03:54:29.807229+0000 mgr.y (mgr.34107) 1281 : cluster 0 pgmap v2005: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:31.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:30 smithi189 bash[66969]: cluster 2024-03-07T03:54:29.807229+0000 mgr.y (mgr.34107) 1281 : cluster 0 pgmap v2005: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:32.590 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:54:32 smithi189 bash[59814]: ts=2024-03-07T03:54:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:54:33.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:54:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:54:32] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2024-03-07T03:54:33.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:32 smithi145 bash[79878]: cluster 2024-03-07T03:54:31.808028+0000 mgr.y (mgr.34107) 1282 : cluster 0 pgmap v2006: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:33.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:32 smithi145 bash[79878]: audit 2024-03-07T03:54:32.692675+0000 mon.a (mon.0) 2042 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:33.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:32 smithi145 bash[82329]: cluster 2024-03-07T03:54:31.808028+0000 mgr.y (mgr.34107) 1282 : cluster 0 pgmap v2006: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:33.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:32 smithi145 bash[82329]: audit 2024-03-07T03:54:32.692675+0000 mon.a (mon.0) 2042 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:33.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:32 smithi189 bash[66969]: cluster 2024-03-07T03:54:31.808028+0000 mgr.y (mgr.34107) 1282 : cluster 0 pgmap v2006: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:33.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:32 smithi189 bash[66969]: audit 2024-03-07T03:54:32.692675+0000 mon.a (mon.0) 2042 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:33.801 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:54:35.212 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:34 smithi145 bash[79878]: cluster 2024-03-07T03:54:33.808801+0000 mgr.y (mgr.34107) 1283 : cluster 0 pgmap v2007: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:35.212 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:34 smithi145 bash[82329]: cluster 2024-03-07T03:54:33.808801+0000 mgr.y (mgr.34107) 1283 : cluster 0 pgmap v2007: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:35.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:34 smithi189 bash[66969]: cluster 2024-03-07T03:54:33.808801+0000 mgr.y (mgr.34107) 1283 : cluster 0 pgmap v2007: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:37.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:36 smithi145 bash[79878]: cluster 2024-03-07T03:54:35.809520+0000 mgr.y (mgr.34107) 1284 : cluster 0 pgmap v2008: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:37.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:36 smithi145 bash[82329]: cluster 2024-03-07T03:54:35.809520+0000 mgr.y (mgr.34107) 1284 : cluster 0 pgmap v2008: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:37.133 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:54:37.133 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:54:37.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:36 smithi189 bash[66969]: cluster 2024-03-07T03:54:35.809520+0000 mgr.y (mgr.34107) 1284 : cluster 0 pgmap v2008: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:38.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:37 smithi145 bash[79878]: audit 2024-03-07T03:54:37.134495+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.145:0/1316196433' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:38.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:37 smithi145 bash[82329]: audit 2024-03-07T03:54:37.134495+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.145:0/1316196433' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:38.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:37 smithi189 bash[66969]: audit 2024-03-07T03:54:37.134495+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.145:0/1316196433' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:39.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:38 smithi145 bash[79878]: cluster 2024-03-07T03:54:37.810361+0000 mgr.y (mgr.34107) 1285 : cluster 0 pgmap v2009: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:39.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:38 smithi145 bash[82329]: cluster 2024-03-07T03:54:37.810361+0000 mgr.y (mgr.34107) 1285 : cluster 0 pgmap v2009: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:39.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:38 smithi189 bash[66969]: cluster 2024-03-07T03:54:37.810361+0000 mgr.y (mgr.34107) 1285 : cluster 0 pgmap v2009: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:40.924 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:54:41.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:40 smithi145 bash[79878]: cluster 2024-03-07T03:54:39.811200+0000 mgr.y (mgr.34107) 1286 : cluster 0 pgmap v2010: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:41.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:40 smithi145 bash[82329]: cluster 2024-03-07T03:54:39.811200+0000 mgr.y (mgr.34107) 1286 : cluster 0 pgmap v2010: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:41.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:40 smithi189 bash[66969]: cluster 2024-03-07T03:54:39.811200+0000 mgr.y (mgr.34107) 1286 : cluster 0 pgmap v2010: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:42.591 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:54:42 smithi189 bash[59814]: ts=2024-03-07T03:54:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:54:43.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:42 smithi145 bash[79878]: cluster 2024-03-07T03:54:41.812154+0000 mgr.y (mgr.34107) 1287 : cluster 0 pgmap v2011: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:43.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:54:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:54:42] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2024-03-07T03:54:43.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:42 smithi145 bash[82329]: cluster 2024-03-07T03:54:41.812154+0000 mgr.y (mgr.34107) 1287 : cluster 0 pgmap v2011: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:43.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:42 smithi189 bash[66969]: cluster 2024-03-07T03:54:41.812154+0000 mgr.y (mgr.34107) 1287 : cluster 0 pgmap v2011: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:45.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:44 smithi145 bash[79878]: cluster 2024-03-07T03:54:43.813009+0000 mgr.y (mgr.34107) 1288 : cluster 0 pgmap v2012: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:45.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:44 smithi145 bash[82329]: cluster 2024-03-07T03:54:43.813009+0000 mgr.y (mgr.34107) 1288 : cluster 0 pgmap v2012: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:45.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:44 smithi189 bash[66969]: cluster 2024-03-07T03:54:43.813009+0000 mgr.y (mgr.34107) 1288 : cluster 0 pgmap v2012: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:45.661 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:54:47.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:46 smithi145 bash[79878]: cluster 2024-03-07T03:54:45.813868+0000 mgr.y (mgr.34107) 1289 : cluster 0 pgmap v2013: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:47.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:46 smithi145 bash[82329]: cluster 2024-03-07T03:54:45.813868+0000 mgr.y (mgr.34107) 1289 : cluster 0 pgmap v2013: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:47.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:46 smithi189 bash[66969]: cluster 2024-03-07T03:54:45.813868+0000 mgr.y (mgr.34107) 1289 : cluster 0 pgmap v2013: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:48.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:47 smithi145 bash[79878]: audit 2024-03-07T03:54:47.692642+0000 mon.a (mon.0) 2043 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:48.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:47 smithi145 bash[82329]: audit 2024-03-07T03:54:47.692642+0000 mon.a (mon.0) 2043 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:48.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:47 smithi189 bash[66969]: audit 2024-03-07T03:54:47.692642+0000 mon.a (mon.0) 2043 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:49.010 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:54:49.010 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:54:49.235 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:48 smithi145 bash[79878]: cluster 2024-03-07T03:54:47.814710+0000 mgr.y (mgr.34107) 1290 : cluster 0 pgmap v2014: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:49.235 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:48 smithi145 bash[82329]: cluster 2024-03-07T03:54:47.814710+0000 mgr.y (mgr.34107) 1290 : cluster 0 pgmap v2014: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:49.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:48 smithi189 bash[66969]: cluster 2024-03-07T03:54:47.814710+0000 mgr.y (mgr.34107) 1290 : cluster 0 pgmap v2014: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:50.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:49 smithi145 bash[79878]: audit 2024-03-07T03:54:49.011714+0000 mon.a (mon.0) 2044 : audit 0 from='client.? 172.21.15.145:0/412023329' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:50.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:49 smithi145 bash[82329]: audit 2024-03-07T03:54:49.011714+0000 mon.a (mon.0) 2044 : audit 0 from='client.? 172.21.15.145:0/412023329' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:50.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:49 smithi189 bash[66969]: audit 2024-03-07T03:54:49.011714+0000 mon.a (mon.0) 2044 : audit 0 from='client.? 172.21.15.145:0/412023329' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:51.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:50 smithi145 bash[79878]: cluster 2024-03-07T03:54:49.815534+0000 mgr.y (mgr.34107) 1291 : cluster 0 pgmap v2015: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:51.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:50 smithi145 bash[82329]: cluster 2024-03-07T03:54:49.815534+0000 mgr.y (mgr.34107) 1291 : cluster 0 pgmap v2015: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:51.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:50 smithi189 bash[66969]: cluster 2024-03-07T03:54:49.815534+0000 mgr.y (mgr.34107) 1291 : cluster 0 pgmap v2015: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:52.591 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:54:52 smithi189 bash[59814]: ts=2024-03-07T03:54:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:54:52.763 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:54:53.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:52 smithi145 bash[79878]: cluster 2024-03-07T03:54:51.816499+0000 mgr.y (mgr.34107) 1292 : cluster 0 pgmap v2016: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:53.071 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:54:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:54:52] "GET /metrics HTTP/1.1" 200 34153 "" "Prometheus/2.43.0" 2024-03-07T03:54:53.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:52 smithi145 bash[82329]: cluster 2024-03-07T03:54:51.816499+0000 mgr.y (mgr.34107) 1292 : cluster 0 pgmap v2016: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:53.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:52 smithi189 bash[66969]: cluster 2024-03-07T03:54:51.816499+0000 mgr.y (mgr.34107) 1292 : cluster 0 pgmap v2016: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:55.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:54 smithi145 bash[79878]: cluster 2024-03-07T03:54:53.817417+0000 mgr.y (mgr.34107) 1293 : cluster 0 pgmap v2017: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:55.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:54 smithi145 bash[82329]: cluster 2024-03-07T03:54:53.817417+0000 mgr.y (mgr.34107) 1293 : cluster 0 pgmap v2017: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:55.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:54 smithi189 bash[66969]: cluster 2024-03-07T03:54:53.817417+0000 mgr.y (mgr.34107) 1293 : cluster 0 pgmap v2017: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:57.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:56 smithi145 bash[79878]: cluster 2024-03-07T03:54:55.818420+0000 mgr.y (mgr.34107) 1294 : cluster 0 pgmap v2018: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:57.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:56 smithi145 bash[82329]: cluster 2024-03-07T03:54:55.818420+0000 mgr.y (mgr.34107) 1294 : cluster 0 pgmap v2018: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:57.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:56 smithi189 bash[66969]: cluster 2024-03-07T03:54:55.818420+0000 mgr.y (mgr.34107) 1294 : cluster 0 pgmap v2018: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:57.503 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:54:59.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:54:58 smithi145 bash[79878]: cluster 2024-03-07T03:54:57.819378+0000 mgr.y (mgr.34107) 1295 : cluster 0 pgmap v2019: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:59.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:54:58 smithi145 bash[82329]: cluster 2024-03-07T03:54:57.819378+0000 mgr.y (mgr.34107) 1295 : cluster 0 pgmap v2019: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:59.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:54:58 smithi189 bash[66969]: cluster 2024-03-07T03:54:57.819378+0000 mgr.y (mgr.34107) 1295 : cluster 0 pgmap v2019: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:00.785 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:55:00.785 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:55:01.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:00 smithi145 bash[82329]: cluster 2024-03-07T03:54:59.820350+0000 mgr.y (mgr.34107) 1296 : cluster 0 pgmap v2020: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:01.058 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:00 smithi145 bash[82329]: audit 2024-03-07T03:55:00.786370+0000 mon.c (mon.2) 67 : audit 0 from='client.? 172.21.15.145:0/2197670114' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:01.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:00 smithi145 bash[79878]: cluster 2024-03-07T03:54:59.820350+0000 mgr.y (mgr.34107) 1296 : cluster 0 pgmap v2020: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:01.059 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:00 smithi145 bash[79878]: audit 2024-03-07T03:55:00.786370+0000 mon.c (mon.2) 67 : audit 0 from='client.? 172.21.15.145:0/2197670114' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:01.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:00 smithi189 bash[66969]: cluster 2024-03-07T03:54:59.820350+0000 mgr.y (mgr.34107) 1296 : cluster 0 pgmap v2020: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:01.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:00 smithi189 bash[66969]: audit 2024-03-07T03:55:00.786370+0000 mon.c (mon.2) 67 : audit 0 from='client.? 172.21.15.145:0/2197670114' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:02.591 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:55:02 smithi189 bash[59814]: ts=2024-03-07T03:55:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:55:03.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:02 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:55:02] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2024-03-07T03:55:03.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:02 smithi145 bash[79878]: cluster 2024-03-07T03:55:01.821472+0000 mgr.y (mgr.34107) 1297 : cluster 0 pgmap v2021: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:03.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:02 smithi145 bash[79878]: audit 2024-03-07T03:55:02.693512+0000 mon.a (mon.0) 2045 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:02 smithi145 bash[82329]: cluster 2024-03-07T03:55:01.821472+0000 mgr.y (mgr.34107) 1297 : cluster 0 pgmap v2021: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:03.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:02 smithi145 bash[82329]: audit 2024-03-07T03:55:02.693512+0000 mon.a (mon.0) 2045 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:03.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:02 smithi189 bash[66969]: cluster 2024-03-07T03:55:01.821472+0000 mgr.y (mgr.34107) 1297 : cluster 0 pgmap v2021: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:03.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:02 smithi189 bash[66969]: audit 2024-03-07T03:55:02.693512+0000 mon.a (mon.0) 2045 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:04.635 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:55:05.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:04 smithi145 bash[79878]: cluster 2024-03-07T03:55:03.822448+0000 mgr.y (mgr.34107) 1298 : cluster 0 pgmap v2022: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:05.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:04 smithi145 bash[82329]: cluster 2024-03-07T03:55:03.822448+0000 mgr.y (mgr.34107) 1298 : cluster 0 pgmap v2022: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:05.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:04 smithi189 bash[66969]: cluster 2024-03-07T03:55:03.822448+0000 mgr.y (mgr.34107) 1298 : cluster 0 pgmap v2022: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:07.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:06 smithi145 bash[79878]: cluster 2024-03-07T03:55:05.823293+0000 mgr.y (mgr.34107) 1299 : cluster 0 pgmap v2023: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:07.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:06 smithi145 bash[82329]: cluster 2024-03-07T03:55:05.823293+0000 mgr.y (mgr.34107) 1299 : cluster 0 pgmap v2023: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:07.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:06 smithi189 bash[66969]: cluster 2024-03-07T03:55:05.823293+0000 mgr.y (mgr.34107) 1299 : cluster 0 pgmap v2023: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:09.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:08 smithi145 bash[79878]: cluster 2024-03-07T03:55:07.824126+0000 mgr.y (mgr.34107) 1300 : cluster 0 pgmap v2024: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:09.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:08 smithi145 bash[82329]: cluster 2024-03-07T03:55:07.824126+0000 mgr.y (mgr.34107) 1300 : cluster 0 pgmap v2024: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:09.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:08 smithi189 bash[66969]: cluster 2024-03-07T03:55:07.824126+0000 mgr.y (mgr.34107) 1300 : cluster 0 pgmap v2024: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:09.377 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:55:11.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:10 smithi145 bash[79878]: cluster 2024-03-07T03:55:09.825018+0000 mgr.y (mgr.34107) 1301 : cluster 0 pgmap v2025: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:11.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:10 smithi145 bash[82329]: cluster 2024-03-07T03:55:09.825018+0000 mgr.y (mgr.34107) 1301 : cluster 0 pgmap v2025: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:11.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:10 smithi189 bash[66969]: cluster 2024-03-07T03:55:09.825018+0000 mgr.y (mgr.34107) 1301 : cluster 0 pgmap v2025: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:12.591 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:55:12 smithi189 bash[59814]: ts=2024-03-07T03:55:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:55:12.902 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:55:12.903 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:55:13.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:12 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:55:12] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2024-03-07T03:55:13.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:12 smithi145 bash[79878]: cluster 2024-03-07T03:55:11.826016+0000 mgr.y (mgr.34107) 1302 : cluster 0 pgmap v2026: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:13.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:12 smithi145 bash[79878]: audit 2024-03-07T03:55:12.903885+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.145:0/4291022266' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:13.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:12 smithi145 bash[82329]: cluster 2024-03-07T03:55:11.826016+0000 mgr.y (mgr.34107) 1302 : cluster 0 pgmap v2026: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:13.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:12 smithi145 bash[82329]: audit 2024-03-07T03:55:12.903885+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.145:0/4291022266' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:13.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:12 smithi189 bash[66969]: cluster 2024-03-07T03:55:11.826016+0000 mgr.y (mgr.34107) 1302 : cluster 0 pgmap v2026: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:13.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:12 smithi189 bash[66969]: audit 2024-03-07T03:55:12.903885+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.145:0/4291022266' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:14.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:14 smithi145 bash[79878]: audit 2024-03-07T03:55:13.888984+0000 mon.a (mon.0) 2046 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:55:14.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:14 smithi145 bash[82329]: audit 2024-03-07T03:55:13.888984+0000 mon.a (mon.0) 2046 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:55:14.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:14 smithi189 bash[66969]: audit 2024-03-07T03:55:13.888984+0000 mon.a (mon.0) 2046 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:55:15.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:15 smithi145 bash[79878]: cluster 2024-03-07T03:55:13.826848+0000 mgr.y (mgr.34107) 1303 : cluster 0 pgmap v2027: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:15.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:15 smithi145 bash[82329]: cluster 2024-03-07T03:55:13.826848+0000 mgr.y (mgr.34107) 1303 : cluster 0 pgmap v2027: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:15.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:15 smithi189 bash[66969]: cluster 2024-03-07T03:55:13.826848+0000 mgr.y (mgr.34107) 1303 : cluster 0 pgmap v2027: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:16.791 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:55:17.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:17 smithi145 bash[79878]: cluster 2024-03-07T03:55:15.827808+0000 mgr.y (mgr.34107) 1304 : cluster 0 pgmap v2028: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:17.070 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:17 smithi145 bash[82329]: cluster 2024-03-07T03:55:15.827808+0000 mgr.y (mgr.34107) 1304 : cluster 0 pgmap v2028: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:17.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:17 smithi189 bash[66969]: cluster 2024-03-07T03:55:15.827808+0000 mgr.y (mgr.34107) 1304 : cluster 0 pgmap v2028: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:18.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:18 smithi145 bash[79878]: audit 2024-03-07T03:55:17.693923+0000 mon.a (mon.0) 2047 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:18.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:18 smithi145 bash[82329]: audit 2024-03-07T03:55:17.693923+0000 mon.a (mon.0) 2047 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:18.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:18 smithi189 bash[66969]: audit 2024-03-07T03:55:17.693923+0000 mon.a (mon.0) 2047 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:19.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:19 smithi145 bash[82329]: cluster 2024-03-07T03:55:17.828709+0000 mgr.y (mgr.34107) 1305 : cluster 0 pgmap v2029: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:19.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:19 smithi145 bash[79878]: cluster 2024-03-07T03:55:17.828709+0000 mgr.y (mgr.34107) 1305 : cluster 0 pgmap v2029: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:19.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:19 smithi189 bash[66969]: cluster 2024-03-07T03:55:17.828709+0000 mgr.y (mgr.34107) 1305 : cluster 0 pgmap v2029: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:20.972 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:55:20.995 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[79878]: audit 2024-03-07T03:55:19.734779+0000 mon.a (mon.0) 2048 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:20.996 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[79878]: audit 2024-03-07T03:55:19.747385+0000 mon.a (mon.0) 2049 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:20.996 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[79878]: cluster 2024-03-07T03:55:19.829327+0000 mgr.y (mgr.34107) 1306 : cluster 0 pgmap v2030: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:20.996 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[79878]: audit 2024-03-07T03:55:19.948120+0000 mon.a (mon.0) 2050 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:20.996 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[79878]: audit 2024-03-07T03:55:19.960657+0000 mon.a (mon.0) 2051 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:20.996 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[79878]: audit 2024-03-07T03:55:20.605292+0000 mon.a (mon.0) 2052 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:55:20.996 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[79878]: audit 2024-03-07T03:55:20.607013+0000 mon.a (mon.0) 2053 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:55:20.996 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[79878]: audit 2024-03-07T03:55:20.616952+0000 mon.a (mon.0) 2054 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:20.996 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[82329]: audit 2024-03-07T03:55:19.734779+0000 mon.a (mon.0) 2048 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:20.996 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[82329]: audit 2024-03-07T03:55:19.747385+0000 mon.a (mon.0) 2049 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:20.997 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[82329]: cluster 2024-03-07T03:55:19.829327+0000 mgr.y (mgr.34107) 1306 : cluster 0 pgmap v2030: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:20.997 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[82329]: audit 2024-03-07T03:55:19.948120+0000 mon.a (mon.0) 2050 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:20.997 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[82329]: audit 2024-03-07T03:55:19.960657+0000 mon.a (mon.0) 2051 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:20.997 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[82329]: audit 2024-03-07T03:55:20.605292+0000 mon.a (mon.0) 2052 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:55:20.997 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[82329]: audit 2024-03-07T03:55:20.607013+0000 mon.a (mon.0) 2053 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:55:20.997 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:20 smithi145 bash[82329]: audit 2024-03-07T03:55:20.616952+0000 mon.a (mon.0) 2054 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:21.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:20 smithi189 bash[66969]: audit 2024-03-07T03:55:19.734779+0000 mon.a (mon.0) 2048 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:21.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:20 smithi189 bash[66969]: audit 2024-03-07T03:55:19.747385+0000 mon.a (mon.0) 2049 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:21.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:20 smithi189 bash[66969]: cluster 2024-03-07T03:55:19.829327+0000 mgr.y (mgr.34107) 1306 : cluster 0 pgmap v2030: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:21.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:20 smithi189 bash[66969]: audit 2024-03-07T03:55:19.948120+0000 mon.a (mon.0) 2050 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:21.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:20 smithi189 bash[66969]: audit 2024-03-07T03:55:19.960657+0000 mon.a (mon.0) 2051 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:21.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:20 smithi189 bash[66969]: audit 2024-03-07T03:55:20.605292+0000 mon.a (mon.0) 2052 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:55:21.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:20 smithi189 bash[66969]: audit 2024-03-07T03:55:20.607013+0000 mon.a (mon.0) 2053 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:55:21.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:20 smithi189 bash[66969]: audit 2024-03-07T03:55:20.616952+0000 mon.a (mon.0) 2054 : audit 1 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' 2024-03-07T03:55:22.591 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:55:22 smithi189 bash[59814]: ts=2024-03-07T03:55:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:55:23.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:22 smithi145 bash[79878]: cluster 2024-03-07T03:55:21.830325+0000 mgr.y (mgr.34107) 1307 : cluster 0 pgmap v2031: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:23.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:22 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:55:22] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2024-03-07T03:55:23.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:22 smithi145 bash[82329]: cluster 2024-03-07T03:55:21.830325+0000 mgr.y (mgr.34107) 1307 : cluster 0 pgmap v2031: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:23.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:22 smithi189 bash[66969]: cluster 2024-03-07T03:55:21.830325+0000 mgr.y (mgr.34107) 1307 : cluster 0 pgmap v2031: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:24.378 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:55:24.378 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:55:25.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:24 smithi145 bash[79878]: cluster 2024-03-07T03:55:23.831177+0000 mgr.y (mgr.34107) 1308 : cluster 0 pgmap v2032: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:25.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:24 smithi145 bash[79878]: audit 2024-03-07T03:55:24.379118+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.145:0/3934824459' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:25.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:24 smithi145 bash[82329]: cluster 2024-03-07T03:55:23.831177+0000 mgr.y (mgr.34107) 1308 : cluster 0 pgmap v2032: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:25.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:24 smithi145 bash[82329]: audit 2024-03-07T03:55:24.379118+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.145:0/3934824459' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:25.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:24 smithi189 bash[66969]: cluster 2024-03-07T03:55:23.831177+0000 mgr.y (mgr.34107) 1308 : cluster 0 pgmap v2032: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:25.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:24 smithi189 bash[66969]: audit 2024-03-07T03:55:24.379118+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.145:0/3934824459' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:27.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:26 smithi145 bash[79878]: cluster 2024-03-07T03:55:25.832183+0000 mgr.y (mgr.34107) 1309 : cluster 0 pgmap v2033: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:27.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:26 smithi145 bash[82329]: cluster 2024-03-07T03:55:25.832183+0000 mgr.y (mgr.34107) 1309 : cluster 0 pgmap v2033: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:27.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:26 smithi189 bash[66969]: cluster 2024-03-07T03:55:25.832183+0000 mgr.y (mgr.34107) 1309 : cluster 0 pgmap v2033: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:28.211 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:55:29.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:28 smithi145 bash[79878]: cluster 2024-03-07T03:55:27.833147+0000 mgr.y (mgr.34107) 1310 : cluster 0 pgmap v2034: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:29.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:28 smithi145 bash[82329]: cluster 2024-03-07T03:55:27.833147+0000 mgr.y (mgr.34107) 1310 : cluster 0 pgmap v2034: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:29.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:28 smithi189 bash[66969]: cluster 2024-03-07T03:55:27.833147+0000 mgr.y (mgr.34107) 1310 : cluster 0 pgmap v2034: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:31.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:30 smithi145 bash[79878]: cluster 2024-03-07T03:55:29.834195+0000 mgr.y (mgr.34107) 1311 : cluster 0 pgmap v2035: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:31.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:30 smithi145 bash[82329]: cluster 2024-03-07T03:55:29.834195+0000 mgr.y (mgr.34107) 1311 : cluster 0 pgmap v2035: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:31.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:30 smithi189 bash[66969]: cluster 2024-03-07T03:55:29.834195+0000 mgr.y (mgr.34107) 1311 : cluster 0 pgmap v2035: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:32.591 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:55:32 smithi189 bash[59814]: ts=2024-03-07T03:55:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:55:32.948 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:55:33.070 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:32 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:55:32] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2024-03-07T03:55:33.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:32 smithi145 bash[79878]: cluster 2024-03-07T03:55:31.835267+0000 mgr.y (mgr.34107) 1312 : cluster 0 pgmap v2036: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:33.071 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:32 smithi145 bash[79878]: audit 2024-03-07T03:55:32.694419+0000 mon.a (mon.0) 2055 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:33.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:32 smithi145 bash[82329]: cluster 2024-03-07T03:55:31.835267+0000 mgr.y (mgr.34107) 1312 : cluster 0 pgmap v2036: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:33.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:32 smithi145 bash[82329]: audit 2024-03-07T03:55:32.694419+0000 mon.a (mon.0) 2055 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:33.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:32 smithi189 bash[66969]: cluster 2024-03-07T03:55:31.835267+0000 mgr.y (mgr.34107) 1312 : cluster 0 pgmap v2036: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:33.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:32 smithi189 bash[66969]: audit 2024-03-07T03:55:32.694419+0000 mon.a (mon.0) 2055 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:35.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:34 smithi145 bash[79878]: cluster 2024-03-07T03:55:33.836243+0000 mgr.y (mgr.34107) 1313 : cluster 0 pgmap v2037: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:35.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:34 smithi145 bash[82329]: cluster 2024-03-07T03:55:33.836243+0000 mgr.y (mgr.34107) 1313 : cluster 0 pgmap v2037: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:35.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:34 smithi189 bash[66969]: cluster 2024-03-07T03:55:33.836243+0000 mgr.y (mgr.34107) 1313 : cluster 0 pgmap v2037: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:36.186 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:55:36.186 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:55:37.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:36 smithi145 bash[79878]: cluster 2024-03-07T03:55:35.836852+0000 mgr.y (mgr.34107) 1314 : cluster 0 pgmap v2038: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:37.321 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:36 smithi145 bash[79878]: audit 2024-03-07T03:55:36.186809+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.145:0/4016876502' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:37.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:36 smithi145 bash[82329]: cluster 2024-03-07T03:55:35.836852+0000 mgr.y (mgr.34107) 1314 : cluster 0 pgmap v2038: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:37.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:36 smithi145 bash[82329]: audit 2024-03-07T03:55:36.186809+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.145:0/4016876502' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:37.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:36 smithi189 bash[66969]: cluster 2024-03-07T03:55:35.836852+0000 mgr.y (mgr.34107) 1314 : cluster 0 pgmap v2038: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:37.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:36 smithi189 bash[66969]: audit 2024-03-07T03:55:36.186809+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.145:0/4016876502' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:39.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:38 smithi145 bash[79878]: cluster 2024-03-07T03:55:37.837659+0000 mgr.y (mgr.34107) 1315 : cluster 0 pgmap v2039: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:39.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:38 smithi145 bash[82329]: cluster 2024-03-07T03:55:37.837659+0000 mgr.y (mgr.34107) 1315 : cluster 0 pgmap v2039: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:39.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:38 smithi189 bash[66969]: cluster 2024-03-07T03:55:37.837659+0000 mgr.y (mgr.34107) 1315 : cluster 0 pgmap v2039: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:39.925 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:55:41.320 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:40 smithi145 bash[82329]: cluster 2024-03-07T03:55:39.838632+0000 mgr.y (mgr.34107) 1316 : cluster 0 pgmap v2040: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:41.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:40 smithi145 bash[79878]: cluster 2024-03-07T03:55:39.838632+0000 mgr.y (mgr.34107) 1316 : cluster 0 pgmap v2040: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:41.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:40 smithi189 bash[66969]: cluster 2024-03-07T03:55:39.838632+0000 mgr.y (mgr.34107) 1316 : cluster 0 pgmap v2040: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:42.591 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:55:42 smithi189 bash[59814]: ts=2024-03-07T03:55:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:55:43.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:42 smithi145 bash[79878]: cluster 2024-03-07T03:55:41.839652+0000 mgr.y (mgr.34107) 1317 : cluster 0 pgmap v2041: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:43.071 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:42 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:55:42] "GET /metrics HTTP/1.1" 200 34152 "" "Prometheus/2.43.0" 2024-03-07T03:55:43.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:42 smithi145 bash[82329]: cluster 2024-03-07T03:55:41.839652+0000 mgr.y (mgr.34107) 1317 : cluster 0 pgmap v2041: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:43.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:42 smithi189 bash[66969]: cluster 2024-03-07T03:55:41.839652+0000 mgr.y (mgr.34107) 1317 : cluster 0 pgmap v2041: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:44.660 INFO:teuthology.orchestra.run.smithi145.stderr:Inferring config /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:55:45.320 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:44 smithi145 bash[79878]: cluster 2024-03-07T03:55:43.840492+0000 mgr.y (mgr.34107) 1318 : cluster 0 pgmap v2042: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:45.321 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:44 smithi145 bash[82329]: cluster 2024-03-07T03:55:43.840492+0000 mgr.y (mgr.34107) 1318 : cluster 0 pgmap v2042: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:45.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:44 smithi189 bash[66969]: cluster 2024-03-07T03:55:43.840492+0000 mgr.y (mgr.34107) 1318 : cluster 0 pgmap v2042: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:47.070 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:46 smithi145 bash[79878]: cluster 2024-03-07T03:55:45.841025+0000 mgr.y (mgr.34107) 1319 : cluster 0 pgmap v2043: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:47.071 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:46 smithi145 bash[82329]: cluster 2024-03-07T03:55:45.841025+0000 mgr.y (mgr.34107) 1319 : cluster 0 pgmap v2043: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:47.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:46 smithi189 bash[66969]: cluster 2024-03-07T03:55:45.841025+0000 mgr.y (mgr.34107) 1319 : cluster 0 pgmap v2043: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:48.016 INFO:teuthology.orchestra.run.smithi145.stdout: 2024-03-07T03:55:48.016 INFO:teuthology.orchestra.run.smithi145.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:55:48.249 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:47 smithi145 bash[79878]: audit 2024-03-07T03:55:47.693842+0000 mon.a (mon.0) 2056 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:48.249 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:47 smithi145 bash[82329]: audit 2024-03-07T03:55:47.693842+0000 mon.a (mon.0) 2056 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:48.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:47 smithi189 bash[66969]: audit 2024-03-07T03:55:47.693842+0000 mon.a (mon.0) 2056 : audit 0 from='mgr.34107 172.21.15.145:0/3924997626' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:48.764 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa/tasks/ceph_manager.py", line 3214, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-03-07T03:55:49.022 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=2653ddcde20c4add9c65b519e9b7ccd6 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa/tasks/ceph_manager.py", line 3214, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-03-07T03:55:49.024 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-03-07T03:55:49.034 INFO:tasks.cephadm:Teardown begin 2024-03-07T03:55:49.035 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-07T03:55:49.049 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:48 smithi145 bash[79878]: cluster 2024-03-07T03:55:47.841756+0000 mgr.y (mgr.34107) 1320 : cluster 0 pgmap v2044: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:49.049 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:48 smithi145 bash[79878]: audit 2024-03-07T03:55:48.017565+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.145:0/2811681422' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:49.049 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:48 smithi145 bash[82329]: cluster 2024-03-07T03:55:47.841756+0000 mgr.y (mgr.34107) 1320 : cluster 0 pgmap v2044: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:49.049 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:48 smithi145 bash[82329]: audit 2024-03-07T03:55:48.017565+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.145:0/2811681422' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:49.052 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-07T03:55:49.068 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-03-07T03:55:49.068 DEBUG:teuthology.orchestra.run.smithi145:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-07T03:55:49.101 DEBUG:teuthology.orchestra.run.smithi189:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-07T03:55:49.120 INFO:tasks.cephadm:Stopping all daemons... 2024-03-07T03:55:49.121 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-03-07T03:55:49.121 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.a 2024-03-07T03:55:49.287 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:49 smithi145 systemd[1]: Stopping Ceph mon.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:55:49.302 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:49 smithi145 bash[79878]: debug 2024-03-07T03:55:49.285+0000 7fdb4cee0700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T03:55:49.302 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:49 smithi145 bash[79878]: debug 2024-03-07T03:55:49.285+0000 7fdb4cee0700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-03-07T03:55:49.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:48 smithi189 bash[66969]: cluster 2024-03-07T03:55:47.841756+0000 mgr.y (mgr.34107) 1320 : cluster 0 pgmap v2044: 73 pgs: 73 active+clean; 582 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:49.341 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:48 smithi189 bash[66969]: audit 2024-03-07T03:55:48.017565+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.145:0/2811681422' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:49.570 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:49 smithi145 bash[72238]: [07/Mar/2024:03:55:49] ENGINE Bus STOPPING 2024-03-07T03:55:49.571 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:49 smithi145 bash[72238]: [07/Mar/2024:03:55:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:55:49.571 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:49 smithi145 bash[72238]: [07/Mar/2024:03:55:49] ENGINE Bus STOPPED 2024-03-07T03:55:49.571 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:49 smithi145 bash[72238]: [07/Mar/2024:03:55:49] ENGINE Bus STARTING 2024-03-07T03:55:49.571 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:49 smithi145 bash[72238]: [07/Mar/2024:03:55:49] ENGINE Serving on http://:::9283 2024-03-07T03:55:49.571 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:49 smithi145 bash[72238]: [07/Mar/2024:03:55:49] ENGINE Bus STARTED 2024-03-07T03:55:50.563 INFO:journalctl@ceph.mon.a.smithi145.stdout:Mar 07 03:55:50 smithi145 bash[176904]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mon-a 2024-03-07T03:55:50.567 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.a.service' 2024-03-07T03:55:50.597 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:55:50.597 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-03-07T03:55:50.597 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-03-07T03:55:50.597 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.c 2024-03-07T03:55:50.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:50 smithi145 systemd[1]: Stopping Ceph mon.c for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:55:50.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:50 smithi145 bash[82329]: debug 2024-03-07T03:55:50.769+0000 7f56d314b700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T03:55:50.820 INFO:journalctl@ceph.mon.c.smithi145.stdout:Mar 07 03:55:50 smithi145 bash[82329]: debug 2024-03-07T03:55:50.769+0000 7f56d314b700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-03-07T03:55:50.821 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:50 smithi145 bash[72238]: [07/Mar/2024:03:55:50] ENGINE Bus STOPPING 2024-03-07T03:55:50.821 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:50 smithi145 bash[72238]: [07/Mar/2024:03:55:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:55:50.821 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:50 smithi145 bash[72238]: [07/Mar/2024:03:55:50] ENGINE Bus STOPPED 2024-03-07T03:55:50.821 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:50 smithi145 bash[72238]: [07/Mar/2024:03:55:50] ENGINE Bus STARTING 2024-03-07T03:55:51.320 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:50 smithi145 bash[72238]: [07/Mar/2024:03:55:50] ENGINE Serving on http://:::9283 2024-03-07T03:55:51.321 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:50 smithi145 bash[72238]: [07/Mar/2024:03:55:50] ENGINE Bus STARTED 2024-03-07T03:55:51.736 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.c.service' 2024-03-07T03:55:51.762 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:55:51.762 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-03-07T03:55:51.762 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-03-07T03:55:51.762 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.b 2024-03-07T03:55:52.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:51 smithi189 systemd[1]: Stopping Ceph mon.b for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:55:52.091 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:51 smithi189 bash[66969]: debug 2024-03-07T03:55:51.967+0000 7f5577789700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T03:55:52.092 INFO:journalctl@ceph.mon.b.smithi189.stdout:Mar 07 03:55:51 smithi189 bash[66969]: debug 2024-03-07T03:55:51.967+0000 7f5577789700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-03-07T03:55:52.591 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:55:52 smithi189 bash[59814]: ts=2024-03-07T03:55:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:55:53.003 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mon.b.service' 2024-03-07T03:55:53.035 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:55:53.035 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-03-07T03:55:53.036 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-03-07T03:55:53.036 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.y 2024-03-07T03:55:53.047 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:52 smithi145 bash[72238]: ::ffff:172.21.15.189 - - [07/Mar/2024:03:55:52] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.43.0" 2024-03-07T03:55:53.320 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:53 smithi145 systemd[1]: Stopping Ceph mgr.y for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:55:53.820 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:55:53 smithi145 bash[57902]: ts=2024-03-07T03:55:53.517Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:55:54.170 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.y.service' 2024-03-07T03:55:54.225 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:53 smithi145 bash[177126]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mgr-y 2024-03-07T03:55:54.226 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:54 smithi145 bash[177187]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-mgr-y 2024-03-07T03:55:54.226 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:54 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.y.service: Deactivated successfully. 2024-03-07T03:55:54.226 INFO:journalctl@ceph.mgr.y.smithi145.stdout:Mar 07 03:55:54 smithi145 systemd[1]: Stopped Ceph mgr.y for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:55:54.243 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:55:54.243 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-03-07T03:55:54.244 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-03-07T03:55:54.244 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.x 2024-03-07T03:55:54.591 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Mar 07 03:55:54 smithi189 systemd[1]: Stopping Ceph mgr.x for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:55:55.241 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@mgr.x.service' 2024-03-07T03:55:55.265 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:55:55.265 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-03-07T03:55:55.265 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-03-07T03:55:55.266 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.0 2024-03-07T03:55:55.570 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:55:55 smithi145 systemd[1]: Stopping Ceph osd.0 for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:55:55.571 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:55:55 smithi145 bash[88998]: debug 2024-03-07T03:55:55.401+0000 7ff910ce9700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:55:55.571 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:55:55 smithi145 bash[88998]: debug 2024-03-07T03:55:55.401+0000 7ff910ce9700 -1 osd.0 1101 *** Got signal Terminated *** 2024-03-07T03:55:55.571 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:55:55 smithi145 bash[88998]: debug 2024-03-07T03:55:55.401+0000 7ff910ce9700 -1 osd.0 1101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:56:01.792 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:56:01 smithi145 bash[177225]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-0 2024-03-07T03:56:01.793 INFO:journalctl@ceph.osd.0.smithi145.stdout:Mar 07 03:56:01 smithi145 bash[177288]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-0 2024-03-07T03:56:02.591 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:56:02 smithi189 bash[59814]: ts=2024-03-07T03:56:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi145:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"8947a0f8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi145:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:56:03.820 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:03 smithi145 bash[57902]: ts=2024-03-07T03:56:03.516Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:03.820 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:03 smithi145 bash[57902]: ts=2024-03-07T03:56:03.517Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:04.007 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.0.service' 2024-03-07T03:56:04.037 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:56:04.037 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-03-07T03:56:04.037 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-03-07T03:56:04.037 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.1 2024-03-07T03:56:04.320 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:56:04 smithi145 systemd[1]: Stopping Ceph osd.1 for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:56:04.321 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:56:04 smithi145 bash[93904]: debug 2024-03-07T03:56:04.221+0000 7f3b2aa66700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:56:04.321 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:56:04 smithi145 bash[93904]: debug 2024-03-07T03:56:04.221+0000 7f3b2aa66700 -1 osd.1 1101 *** Got signal Terminated *** 2024-03-07T03:56:04.321 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:56:04 smithi145 bash[93904]: debug 2024-03-07T03:56:04.221+0000 7f3b2aa66700 -1 osd.1 1101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:56:10.538 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:56:10 smithi145 bash[177428]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-1 2024-03-07T03:56:10.538 INFO:journalctl@ceph.osd.1.smithi145.stdout:Mar 07 03:56:10 smithi145 bash[177491]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-1 2024-03-07T03:56:10.539 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:10 smithi145 bash[57902]: ts=2024-03-07T03:56:10.285Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:10.539 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:10 smithi145 bash[57902]: ts=2024-03-07T03:56:10.286Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:56:12.533 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.1.service' 2024-03-07T03:56:12.559 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:56:12.560 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-03-07T03:56:12.560 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-03-07T03:56:12.560 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.2 2024-03-07T03:56:12.820 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:56:12 smithi145 systemd[1]: Stopping Ceph osd.2 for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:56:12.821 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:56:12 smithi145 bash[99521]: debug 2024-03-07T03:56:12.745+0000 7f935f262700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:56:12.821 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:56:12 smithi145 bash[99521]: debug 2024-03-07T03:56:12.745+0000 7f935f262700 -1 osd.2 1101 *** Got signal Terminated *** 2024-03-07T03:56:12.821 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:56:12 smithi145 bash[99521]: debug 2024-03-07T03:56:12.745+0000 7f935f262700 -1 osd.2 1101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:56:13.820 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:13 smithi145 bash[57902]: ts=2024-03-07T03:56:13.517Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:13.820 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:13 smithi145 bash[57902]: ts=2024-03-07T03:56:13.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:19.135 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:56:18 smithi145 bash[177626]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-2 2024-03-07T03:56:19.135 INFO:journalctl@ceph.osd.2.smithi145.stdout:Mar 07 03:56:19 smithi145 bash[177690]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-2 2024-03-07T03:56:20.570 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:20 smithi145 bash[57902]: ts=2024-03-07T03:56:20.284Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:56:20.571 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:20 smithi145 bash[57902]: ts=2024-03-07T03:56:20.285Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:20.571 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:20 smithi145 bash[57902]: ts=2024-03-07T03:56:20.286Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:56:21.099 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.2.service' 2024-03-07T03:56:21.125 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:56:21.125 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-03-07T03:56:21.125 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-03-07T03:56:21.125 DEBUG:teuthology.orchestra.run.smithi145:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.3 2024-03-07T03:56:21.571 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:56:21 smithi145 systemd[1]: Stopping Ceph osd.3 for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:56:21.571 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:56:21 smithi145 bash[103903]: debug 2024-03-07T03:56:21.309+0000 7ffb01b23700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:56:21.571 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:56:21 smithi145 bash[103903]: debug 2024-03-07T03:56:21.309+0000 7ffb01b23700 -1 osd.3 1101 *** Got signal Terminated *** 2024-03-07T03:56:21.571 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:56:21 smithi145 bash[103903]: debug 2024-03-07T03:56:21.309+0000 7ffb01b23700 -1 osd.3 1101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:56:21.803 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:21 smithi189 bash[43534]: debug 2024-03-07T03:56:21.511+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:22.091 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:22 smithi189 bash[72532]: debug 2024-03-07T03:56:22.023+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:58.427154+0000 front 2024-03-07T03:55:58.426653+0000 (oldest deadline 2024-03-07T03:56:21.926715+0000) 2024-03-07T03:56:22.092 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:21 smithi189 bash[32451]: debug 2024-03-07T03:56:21.799+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:22.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:22 smithi189 bash[43534]: debug 2024-03-07T03:56:22.507+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:23.341 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:23 smithi189 bash[72532]: debug 2024-03-07T03:56:23.043+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:58.427154+0000 front 2024-03-07T03:55:58.426653+0000 (oldest deadline 2024-03-07T03:56:21.926715+0000) 2024-03-07T03:56:23.342 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:22 smithi189 bash[32451]: debug 2024-03-07T03:56:22.847+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:23.342 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:56:22 smithi189 bash[59814]: ts=2024-03-07T03:56:22.931Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-03-07T03:56:23.342 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:56:22 smithi189 bash[59814]: ts=2024-03-07T03:56:22.934Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-03-07T03:56:23.342 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:56:22 smithi189 bash[59814]: ts=2024-03-07T03:56:22.937Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-03-07T03:56:23.342 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:56:22 smithi189 bash[59814]: ts=2024-03-07T03:56:22.937Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-03-07T03:56:23.342 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:56:22 smithi189 bash[59814]: ts=2024-03-07T03:56:22.938Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.145:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.145:8765: connect: connection refused" 2024-03-07T03:56:23.803 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:23 smithi189 bash[43534]: debug 2024-03-07T03:56:23.478+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:23.820 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:23 smithi145 bash[57902]: ts=2024-03-07T03:56:23.517Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:23.820 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:23 smithi145 bash[57902]: ts=2024-03-07T03:56:23.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:24.091 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:24 smithi189 bash[72532]: debug 2024-03-07T03:56:24.010+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:58.427154+0000 front 2024-03-07T03:55:58.426653+0000 (oldest deadline 2024-03-07T03:56:21.926715+0000) 2024-03-07T03:56:24.092 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:23 smithi189 bash[32451]: debug 2024-03-07T03:56:23.798+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:24.570 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:56:24 smithi145 bash[103903]: debug 2024-03-07T03:56:24.165+0000 7ffafa10f700 -1 osd.3 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.125685+0000 front 2024-03-07T03:56:00.125886+0000 (oldest deadline 2024-03-07T03:56:23.625858+0000) 2024-03-07T03:56:24.795 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:24 smithi189 bash[43534]: debug 2024-03-07T03:56:24.522+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:25.091 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:25 smithi189 bash[72532]: debug 2024-03-07T03:56:25.038+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:58.427154+0000 front 2024-03-07T03:55:58.426653+0000 (oldest deadline 2024-03-07T03:56:21.926715+0000) 2024-03-07T03:56:25.092 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:24 smithi189 bash[32451]: debug 2024-03-07T03:56:24.790+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:25.570 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:56:25 smithi145 bash[103903]: debug 2024-03-07T03:56:25.153+0000 7ffafa10f700 -1 osd.3 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.125685+0000 front 2024-03-07T03:56:00.125886+0000 (oldest deadline 2024-03-07T03:56:23.625858+0000) 2024-03-07T03:56:25.768 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:25 smithi189 bash[43534]: debug 2024-03-07T03:56:25.482+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:26.032 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:25 smithi189 bash[32451]: debug 2024-03-07T03:56:25.766+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:26.341 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:26 smithi189 bash[72532]: debug 2024-03-07T03:56:26.026+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:58.427154+0000 front 2024-03-07T03:55:58.426653+0000 (oldest deadline 2024-03-07T03:56:21.926715+0000) 2024-03-07T03:56:26.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:26 smithi189 bash[38071]: debug 2024-03-07T03:56:26.046+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:26.570 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:56:26 smithi145 bash[103903]: debug 2024-03-07T03:56:26.185+0000 7ffafa10f700 -1 osd.3 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.125685+0000 front 2024-03-07T03:56:00.125886+0000 (oldest deadline 2024-03-07T03:56:23.625858+0000) 2024-03-07T03:56:26.802 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:26 smithi189 bash[43534]: debug 2024-03-07T03:56:26.506+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:27.080 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:26 smithi189 bash[32451]: debug 2024-03-07T03:56:26.798+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:27.080 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:27 smithi189 bash[38071]: debug 2024-03-07T03:56:27.018+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:27.341 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:27 smithi189 bash[72532]: debug 2024-03-07T03:56:27.074+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:58.427154+0000 front 2024-03-07T03:55:58.426653+0000 (oldest deadline 2024-03-07T03:56:21.926715+0000) 2024-03-07T03:56:27.570 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:56:27 smithi145 bash[177824]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-3 2024-03-07T03:56:27.570 INFO:journalctl@ceph.osd.3.smithi145.stdout:Mar 07 03:56:27 smithi145 bash[177886]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-3 2024-03-07T03:56:27.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:27 smithi189 bash[43534]: debug 2024-03-07T03:56:27.474+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:28.341 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:28 smithi189 bash[72532]: debug 2024-03-07T03:56:28.058+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:58.427154+0000 front 2024-03-07T03:55:58.426653+0000 (oldest deadline 2024-03-07T03:56:21.926715+0000) 2024-03-07T03:56:28.342 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:27 smithi189 bash[32451]: debug 2024-03-07T03:56:27.838+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:28.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:27 smithi189 bash[38071]: debug 2024-03-07T03:56:27.974+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:28.816 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:28 smithi189 bash[43534]: debug 2024-03-07T03:56:28.490+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:29.091 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:28 smithi189 bash[32451]: debug 2024-03-07T03:56:28.810+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:29.092 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:29 smithi189 bash[72532]: debug 2024-03-07T03:56:29.010+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:58.427154+0000 front 2024-03-07T03:55:58.426653+0000 (oldest deadline 2024-03-07T03:56:21.926715+0000) 2024-03-07T03:56:29.092 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:29 smithi189 bash[38071]: debug 2024-03-07T03:56:29.002+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:29.387 DEBUG:teuthology.orchestra.run.smithi145:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.3.service' 2024-03-07T03:56:29.413 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:56:29.414 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-03-07T03:56:29.414 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-03-07T03:56:29.414 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.4 2024-03-07T03:56:29.833 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:29 smithi189 bash[43534]: debug 2024-03-07T03:56:29.466+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:29.834 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:29 smithi189 systemd[1]: Stopping Ceph osd.4 for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:56:29.834 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:29 smithi189 bash[72532]: debug 2024-03-07T03:56:29.562+0000 7fd7f0ff1700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:56:29.834 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:29 smithi189 bash[72532]: debug 2024-03-07T03:56:29.562+0000 7fd7f0ff1700 -1 osd.4 1101 *** Got signal Terminated *** 2024-03-07T03:56:29.834 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:29 smithi189 bash[72532]: debug 2024-03-07T03:56:29.562+0000 7fd7f0ff1700 -1 osd.4 1101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:56:30.091 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:30 smithi189 bash[38071]: debug 2024-03-07T03:56:30.014+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:30.092 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:30 smithi189 bash[72532]: debug 2024-03-07T03:56:29.998+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:58.427154+0000 front 2024-03-07T03:55:58.426653+0000 (oldest deadline 2024-03-07T03:56:21.926715+0000) 2024-03-07T03:56:30.092 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:29 smithi189 bash[32451]: debug 2024-03-07T03:56:29.830+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:30.570 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:30 smithi145 bash[57902]: ts=2024-03-07T03:56:30.285Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:30.571 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:30 smithi145 bash[57902]: ts=2024-03-07T03:56:30.286Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:30.571 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:30 smithi145 bash[57902]: ts=2024-03-07T03:56:30.288Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:56:30.788 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:30 smithi189 bash[43534]: debug 2024-03-07T03:56:30.502+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:30.788 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:30 smithi189 bash[43534]: debug 2024-03-07T03:56:30.502+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:31.044 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:30 smithi189 bash[32451]: debug 2024-03-07T03:56:30.782+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:31.044 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:30 smithi189 bash[32451]: debug 2024-03-07T03:56:30.782+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.957921+0000 front 2024-03-07T03:56:06.957969+0000 (oldest deadline 2024-03-07T03:56:30.457783+0000) 2024-03-07T03:56:31.044 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:30 smithi189 bash[38071]: debug 2024-03-07T03:56:30.990+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:31.341 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:31 smithi189 bash[72532]: debug 2024-03-07T03:56:31.038+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:58.427154+0000 front 2024-03-07T03:55:58.426653+0000 (oldest deadline 2024-03-07T03:56:21.926715+0000) 2024-03-07T03:56:31.841 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:31 smithi189 bash[32451]: debug 2024-03-07T03:56:31.750+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:31.841 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:31 smithi189 bash[32451]: debug 2024-03-07T03:56:31.750+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.957921+0000 front 2024-03-07T03:56:06.957969+0000 (oldest deadline 2024-03-07T03:56:30.457783+0000) 2024-03-07T03:56:31.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:31 smithi189 bash[43534]: debug 2024-03-07T03:56:31.530+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:31.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:31 smithi189 bash[43534]: debug 2024-03-07T03:56:31.530+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:32.341 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:32 smithi189 bash[72532]: debug 2024-03-07T03:56:32.002+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:58.427154+0000 front 2024-03-07T03:55:58.426653+0000 (oldest deadline 2024-03-07T03:56:21.926715+0000) 2024-03-07T03:56:32.341 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:32 smithi189 bash[72532]: debug 2024-03-07T03:56:32.002+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.028170+0000 front 2024-03-07T03:56:06.028484+0000 (oldest deadline 2024-03-07T03:56:31.328148+0000) 2024-03-07T03:56:32.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:31 smithi189 bash[38071]: debug 2024-03-07T03:56:31.990+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:32.841 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:32 smithi189 bash[32451]: debug 2024-03-07T03:56:32.778+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:32.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:32 smithi189 bash[32451]: debug 2024-03-07T03:56:32.778+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.957921+0000 front 2024-03-07T03:56:06.957969+0000 (oldest deadline 2024-03-07T03:56:30.457783+0000) 2024-03-07T03:56:32.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:32 smithi189 bash[43534]: debug 2024-03-07T03:56:32.546+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:32.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:32 smithi189 bash[43534]: debug 2024-03-07T03:56:32.546+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:33.341 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:33 smithi189 bash[72532]: debug 2024-03-07T03:56:33.042+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:58.427154+0000 front 2024-03-07T03:55:58.426653+0000 (oldest deadline 2024-03-07T03:56:21.926715+0000) 2024-03-07T03:56:33.342 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:33 smithi189 bash[72532]: debug 2024-03-07T03:56:33.042+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.028170+0000 front 2024-03-07T03:56:06.028484+0000 (oldest deadline 2024-03-07T03:56:31.328148+0000) 2024-03-07T03:56:33.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:32 smithi189 bash[38071]: debug 2024-03-07T03:56:32.966+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:33.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:32 smithi189 bash[38071]: debug 2024-03-07T03:56:32.966+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:33.820 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:33 smithi145 bash[57902]: ts=2024-03-07T03:56:33.518Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:33.821 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:33 smithi145 bash[57902]: ts=2024-03-07T03:56:33.519Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:33.841 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:33 smithi189 bash[32451]: debug 2024-03-07T03:56:33.778+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:33.841 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:33 smithi189 bash[32451]: debug 2024-03-07T03:56:33.778+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.957921+0000 front 2024-03-07T03:56:06.957969+0000 (oldest deadline 2024-03-07T03:56:30.457783+0000) 2024-03-07T03:56:33.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:33 smithi189 bash[43534]: debug 2024-03-07T03:56:33.546+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:33.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:33 smithi189 bash[43534]: debug 2024-03-07T03:56:33.546+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:34.341 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:33 smithi189 bash[72532]: debug 2024-03-07T03:56:33.994+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:58.427154+0000 front 2024-03-07T03:55:58.426653+0000 (oldest deadline 2024-03-07T03:56:21.926715+0000) 2024-03-07T03:56:34.341 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:33 smithi189 bash[72532]: debug 2024-03-07T03:56:33.994+0000 7fd7ed5e5700 -1 osd.4 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.028170+0000 front 2024-03-07T03:56:06.028484+0000 (oldest deadline 2024-03-07T03:56:31.328148+0000) 2024-03-07T03:56:34.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:34 smithi189 bash[38071]: debug 2024-03-07T03:56:34.010+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:34.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:34 smithi189 bash[38071]: debug 2024-03-07T03:56:34.010+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:34.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:34 smithi189 bash[43534]: debug 2024-03-07T03:56:34.542+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:34.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:34 smithi189 bash[43534]: debug 2024-03-07T03:56:34.542+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:34.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:34 smithi189 bash[32451]: debug 2024-03-07T03:56:34.766+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:34.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:34 smithi189 bash[32451]: debug 2024-03-07T03:56:34.766+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.957921+0000 front 2024-03-07T03:56:06.957969+0000 (oldest deadline 2024-03-07T03:56:30.457783+0000) 2024-03-07T03:56:35.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:34 smithi189 bash[38071]: debug 2024-03-07T03:56:34.966+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:35.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:34 smithi189 bash[38071]: debug 2024-03-07T03:56:34.966+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:35.788 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:35 smithi189 bash[32451]: debug 2024-03-07T03:56:35.734+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:35.788 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:35 smithi189 bash[32451]: debug 2024-03-07T03:56:35.734+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.957921+0000 front 2024-03-07T03:56:06.957969+0000 (oldest deadline 2024-03-07T03:56:30.457783+0000) 2024-03-07T03:56:35.788 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:35 smithi189 bash[43534]: debug 2024-03-07T03:56:35.534+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:35.788 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:35 smithi189 bash[43534]: debug 2024-03-07T03:56:35.534+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:35.789 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:35 smithi189 bash[90709]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-4 2024-03-07T03:56:35.789 INFO:journalctl@ceph.osd.4.smithi189.stdout:Mar 07 03:56:35 smithi189 bash[90774]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-4 2024-03-07T03:56:36.091 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:36 smithi189 bash[38071]: debug 2024-03-07T03:56:36.006+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:36.091 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:36 smithi189 bash[38071]: debug 2024-03-07T03:56:36.006+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:36.841 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:36 smithi189 bash[32451]: debug 2024-03-07T03:56:36.706+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:36.841 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:36 smithi189 bash[32451]: debug 2024-03-07T03:56:36.706+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.957921+0000 front 2024-03-07T03:56:06.957969+0000 (oldest deadline 2024-03-07T03:56:30.457783+0000) 2024-03-07T03:56:36.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:36 smithi189 bash[43534]: debug 2024-03-07T03:56:36.570+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:36.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:36 smithi189 bash[43534]: debug 2024-03-07T03:56:36.570+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:37.330 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:37 smithi189 bash[38071]: debug 2024-03-07T03:56:37.026+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:37.359 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:37 smithi189 bash[38071]: debug 2024-03-07T03:56:37.026+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:37.769 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.4.service' 2024-03-07T03:56:37.796 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:56:37.796 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-03-07T03:56:37.797 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-03-07T03:56:37.797 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.5 2024-03-07T03:56:37.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:37 smithi189 bash[32451]: debug 2024-03-07T03:56:37.670+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:37.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:37 smithi189 bash[32451]: debug 2024-03-07T03:56:37.670+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.957921+0000 front 2024-03-07T03:56:06.957969+0000 (oldest deadline 2024-03-07T03:56:30.457783+0000) 2024-03-07T03:56:37.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:37 smithi189 bash[43534]: debug 2024-03-07T03:56:37.578+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:37.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:37 smithi189 bash[43534]: debug 2024-03-07T03:56:37.578+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:38.078 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:37 smithi189 systemd[1]: Stopping Ceph osd.5 for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:56:38.078 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:37 smithi189 bash[32451]: debug 2024-03-07T03:56:37.970+0000 7f3747b58700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:56:38.078 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:37 smithi189 bash[32451]: debug 2024-03-07T03:56:37.970+0000 7f3747b58700 -1 osd.5 1101 *** Got signal Terminated *** 2024-03-07T03:56:38.078 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:37 smithi189 bash[32451]: debug 2024-03-07T03:56:37.970+0000 7f3747b58700 -1 osd.5 1101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:56:38.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:38 smithi189 bash[38071]: debug 2024-03-07T03:56:38.074+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:38.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:38 smithi189 bash[38071]: debug 2024-03-07T03:56:38.074+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:38.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:38 smithi189 bash[43534]: debug 2024-03-07T03:56:38.542+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:38.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:38 smithi189 bash[43534]: debug 2024-03-07T03:56:38.542+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:38.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:38 smithi189 bash[32451]: debug 2024-03-07T03:56:38.626+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:38.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:38 smithi189 bash[32451]: debug 2024-03-07T03:56:38.626+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.957921+0000 front 2024-03-07T03:56:06.957969+0000 (oldest deadline 2024-03-07T03:56:30.457783+0000) 2024-03-07T03:56:39.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:39 smithi189 bash[38071]: debug 2024-03-07T03:56:39.030+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:39.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:39 smithi189 bash[38071]: debug 2024-03-07T03:56:39.030+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:39.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:39 smithi189 bash[43534]: debug 2024-03-07T03:56:39.530+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:39.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:39 smithi189 bash[43534]: debug 2024-03-07T03:56:39.530+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:39.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:39 smithi189 bash[43534]: debug 2024-03-07T03:56:39.530+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:39.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:39 smithi189 bash[32451]: debug 2024-03-07T03:56:39.630+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:39.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:39 smithi189 bash[32451]: debug 2024-03-07T03:56:39.630+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.957921+0000 front 2024-03-07T03:56:06.957969+0000 (oldest deadline 2024-03-07T03:56:30.457783+0000) 2024-03-07T03:56:39.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:39 smithi189 bash[32451]: debug 2024-03-07T03:56:39.630+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:13.359044+0000 front 2024-03-07T03:56:13.359127+0000 (oldest deadline 2024-03-07T03:56:39.159833+0000) 2024-03-07T03:56:40.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:40 smithi189 bash[38071]: debug 2024-03-07T03:56:40.058+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:40.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:40 smithi189 bash[38071]: debug 2024-03-07T03:56:40.058+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:40.570 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:40 smithi145 bash[57902]: ts=2024-03-07T03:56:40.286Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:56:40.571 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:40 smithi145 bash[57902]: ts=2024-03-07T03:56:40.287Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:40.571 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:40 smithi145 bash[57902]: ts=2024-03-07T03:56:40.289Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:56:40.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:40 smithi189 bash[43534]: debug 2024-03-07T03:56:40.502+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:40.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:40 smithi189 bash[43534]: debug 2024-03-07T03:56:40.502+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:40.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:40 smithi189 bash[43534]: debug 2024-03-07T03:56:40.502+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:40.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:40 smithi189 bash[32451]: debug 2024-03-07T03:56:40.634+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:40.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:40 smithi189 bash[32451]: debug 2024-03-07T03:56:40.634+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.957921+0000 front 2024-03-07T03:56:06.957969+0000 (oldest deadline 2024-03-07T03:56:30.457783+0000) 2024-03-07T03:56:40.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:40 smithi189 bash[32451]: debug 2024-03-07T03:56:40.634+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:13.359044+0000 front 2024-03-07T03:56:13.359127+0000 (oldest deadline 2024-03-07T03:56:39.159833+0000) 2024-03-07T03:56:41.529 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:41 smithi189 bash[38071]: debug 2024-03-07T03:56:41.094+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:41.529 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:41 smithi189 bash[38071]: debug 2024-03-07T03:56:41.094+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:41.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:41 smithi189 bash[43534]: debug 2024-03-07T03:56:41.526+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:41.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:41 smithi189 bash[43534]: debug 2024-03-07T03:56:41.526+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:41.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:41 smithi189 bash[43534]: debug 2024-03-07T03:56:41.526+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:41.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:41 smithi189 bash[32451]: debug 2024-03-07T03:56:41.654+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:41.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:41 smithi189 bash[32451]: debug 2024-03-07T03:56:41.654+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.957921+0000 front 2024-03-07T03:56:06.957969+0000 (oldest deadline 2024-03-07T03:56:30.457783+0000) 2024-03-07T03:56:41.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:41 smithi189 bash[32451]: debug 2024-03-07T03:56:41.654+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:13.359044+0000 front 2024-03-07T03:56:13.359127+0000 (oldest deadline 2024-03-07T03:56:39.159833+0000) 2024-03-07T03:56:42.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:42 smithi189 bash[38071]: debug 2024-03-07T03:56:42.046+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:42.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:42 smithi189 bash[38071]: debug 2024-03-07T03:56:42.046+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:42.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:42 smithi189 bash[43534]: debug 2024-03-07T03:56:42.490+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:42.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:42 smithi189 bash[43534]: debug 2024-03-07T03:56:42.490+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:42.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:42 smithi189 bash[43534]: debug 2024-03-07T03:56:42.490+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:42.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:42 smithi189 bash[32451]: debug 2024-03-07T03:56:42.686+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:56:00.056003+0000 front 2024-03-07T03:56:00.055521+0000 (oldest deadline 2024-03-07T03:56:21.155639+0000) 2024-03-07T03:56:42.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:42 smithi189 bash[32451]: debug 2024-03-07T03:56:42.686+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.957921+0000 front 2024-03-07T03:56:06.957969+0000 (oldest deadline 2024-03-07T03:56:30.457783+0000) 2024-03-07T03:56:42.842 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:42 smithi189 bash[32451]: debug 2024-03-07T03:56:42.686+0000 7f3740146700 -1 osd.5 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:13.359044+0000 front 2024-03-07T03:56:13.359127+0000 (oldest deadline 2024-03-07T03:56:39.159833+0000) 2024-03-07T03:56:43.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:43 smithi189 bash[38071]: debug 2024-03-07T03:56:43.058+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:43.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:43 smithi189 bash[38071]: debug 2024-03-07T03:56:43.058+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:43.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:43 smithi189 bash[38071]: debug 2024-03-07T03:56:43.058+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.739978+0000 front 2024-03-07T03:56:16.739736+0000 (oldest deadline 2024-03-07T03:56:42.639475+0000) 2024-03-07T03:56:43.820 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:43 smithi145 bash[57902]: ts=2024-03-07T03:56:43.519Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:43.821 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:43 smithi145 bash[57902]: ts=2024-03-07T03:56:43.520Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:43.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:43 smithi189 bash[43534]: debug 2024-03-07T03:56:43.494+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:43.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:43 smithi189 bash[43534]: debug 2024-03-07T03:56:43.494+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:43.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:43 smithi189 bash[43534]: debug 2024-03-07T03:56:43.494+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:44.341 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:43 smithi189 bash[90908]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-5 2024-03-07T03:56:44.341 INFO:journalctl@ceph.osd.5.smithi189.stdout:Mar 07 03:56:44 smithi189 bash[90971]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-5 2024-03-07T03:56:44.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:44 smithi189 bash[38071]: debug 2024-03-07T03:56:44.098+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:44.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:44 smithi189 bash[38071]: debug 2024-03-07T03:56:44.098+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:44.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:44 smithi189 bash[38071]: debug 2024-03-07T03:56:44.098+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.739978+0000 front 2024-03-07T03:56:16.739736+0000 (oldest deadline 2024-03-07T03:56:42.639475+0000) 2024-03-07T03:56:44.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:44 smithi189 bash[43534]: debug 2024-03-07T03:56:44.506+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:44.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:44 smithi189 bash[43534]: debug 2024-03-07T03:56:44.506+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:44.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:44 smithi189 bash[43534]: debug 2024-03-07T03:56:44.506+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:45.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:45 smithi189 bash[38071]: debug 2024-03-07T03:56:45.098+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:45.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:45 smithi189 bash[38071]: debug 2024-03-07T03:56:45.098+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:45.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:45 smithi189 bash[38071]: debug 2024-03-07T03:56:45.098+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.739978+0000 front 2024-03-07T03:56:16.739736+0000 (oldest deadline 2024-03-07T03:56:42.639475+0000) 2024-03-07T03:56:45.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:45 smithi189 bash[43534]: debug 2024-03-07T03:56:45.470+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:45.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:45 smithi189 bash[43534]: debug 2024-03-07T03:56:45.470+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:45.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:45 smithi189 bash[43534]: debug 2024-03-07T03:56:45.470+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:46.017 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.5.service' 2024-03-07T03:56:46.042 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:56:46.043 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-03-07T03:56:46.043 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-03-07T03:56:46.043 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.6 2024-03-07T03:56:46.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:46 smithi189 bash[38071]: debug 2024-03-07T03:56:46.050+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:46.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:46 smithi189 bash[38071]: debug 2024-03-07T03:56:46.050+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:46.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:46 smithi189 bash[38071]: debug 2024-03-07T03:56:46.050+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.739978+0000 front 2024-03-07T03:56:16.739736+0000 (oldest deadline 2024-03-07T03:56:42.639475+0000) 2024-03-07T03:56:46.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:46 smithi189 systemd[1]: Stopping Ceph osd.6 for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:56:46.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:46 smithi189 bash[38071]: debug 2024-03-07T03:56:46.218+0000 7f4ff575c700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:56:46.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:46 smithi189 bash[38071]: debug 2024-03-07T03:56:46.218+0000 7f4ff575c700 -1 osd.6 1101 *** Got signal Terminated *** 2024-03-07T03:56:46.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:46 smithi189 bash[38071]: debug 2024-03-07T03:56:46.218+0000 7f4ff575c700 -1 osd.6 1101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:56:46.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:46 smithi189 bash[43534]: debug 2024-03-07T03:56:46.482+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:46.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:46 smithi189 bash[43534]: debug 2024-03-07T03:56:46.482+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:46.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:46 smithi189 bash[43534]: debug 2024-03-07T03:56:46.482+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:47.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:47 smithi189 bash[38071]: debug 2024-03-07T03:56:47.030+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:47.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:47 smithi189 bash[38071]: debug 2024-03-07T03:56:47.030+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:47.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:47 smithi189 bash[38071]: debug 2024-03-07T03:56:47.030+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.739978+0000 front 2024-03-07T03:56:16.739736+0000 (oldest deadline 2024-03-07T03:56:42.639475+0000) 2024-03-07T03:56:47.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:47 smithi189 bash[43534]: debug 2024-03-07T03:56:47.478+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:47.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:47 smithi189 bash[43534]: debug 2024-03-07T03:56:47.478+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:47.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:47 smithi189 bash[43534]: debug 2024-03-07T03:56:47.478+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:48.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:47 smithi189 bash[38071]: debug 2024-03-07T03:56:47.986+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:48.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:47 smithi189 bash[38071]: debug 2024-03-07T03:56:47.986+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:48.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:47 smithi189 bash[38071]: debug 2024-03-07T03:56:47.986+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.739978+0000 front 2024-03-07T03:56:16.739736+0000 (oldest deadline 2024-03-07T03:56:42.639475+0000) 2024-03-07T03:56:48.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:47 smithi189 bash[38071]: debug 2024-03-07T03:56:47.986+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-03-07T03:56:24.941082+0000 front 2024-03-07T03:56:24.941293+0000 (oldest deadline 2024-03-07T03:56:47.840929+0000) 2024-03-07T03:56:48.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:48 smithi189 bash[43534]: debug 2024-03-07T03:56:48.454+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:48.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:48 smithi189 bash[43534]: debug 2024-03-07T03:56:48.454+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:48.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:48 smithi189 bash[43534]: debug 2024-03-07T03:56:48.454+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:49.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:48 smithi189 bash[38071]: debug 2024-03-07T03:56:48.982+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:49.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:48 smithi189 bash[38071]: debug 2024-03-07T03:56:48.982+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:49.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:48 smithi189 bash[38071]: debug 2024-03-07T03:56:48.982+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.739978+0000 front 2024-03-07T03:56:16.739736+0000 (oldest deadline 2024-03-07T03:56:42.639475+0000) 2024-03-07T03:56:49.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:48 smithi189 bash[38071]: debug 2024-03-07T03:56:48.982+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-03-07T03:56:24.941082+0000 front 2024-03-07T03:56:24.941293+0000 (oldest deadline 2024-03-07T03:56:47.840929+0000) 2024-03-07T03:56:49.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:49 smithi189 bash[43534]: debug 2024-03-07T03:56:49.498+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:49.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:49 smithi189 bash[43534]: debug 2024-03-07T03:56:49.498+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:49.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:49 smithi189 bash[43534]: debug 2024-03-07T03:56:49.498+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:50.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:49 smithi189 bash[38071]: debug 2024-03-07T03:56:49.994+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:50.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:49 smithi189 bash[38071]: debug 2024-03-07T03:56:49.994+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:50.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:49 smithi189 bash[38071]: debug 2024-03-07T03:56:49.994+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.739978+0000 front 2024-03-07T03:56:16.739736+0000 (oldest deadline 2024-03-07T03:56:42.639475+0000) 2024-03-07T03:56:50.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:49 smithi189 bash[38071]: debug 2024-03-07T03:56:49.994+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-03-07T03:56:24.941082+0000 front 2024-03-07T03:56:24.941293+0000 (oldest deadline 2024-03-07T03:56:47.840929+0000) 2024-03-07T03:56:50.570 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:50 smithi145 bash[57902]: ts=2024-03-07T03:56:50.287Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:56:50.571 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:50 smithi145 bash[57902]: ts=2024-03-07T03:56:50.288Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:50.571 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:50 smithi145 bash[57902]: ts=2024-03-07T03:56:50.289Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:56:50.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:50 smithi189 bash[43534]: debug 2024-03-07T03:56:50.482+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:50.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:50 smithi189 bash[43534]: debug 2024-03-07T03:56:50.482+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:50.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:50 smithi189 bash[43534]: debug 2024-03-07T03:56:50.482+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:51.341 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:50 smithi189 bash[38071]: debug 2024-03-07T03:56:50.982+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:59.236645+0000 front 2024-03-07T03:55:59.236417+0000 (oldest deadline 2024-03-07T03:56:25.136251+0000) 2024-03-07T03:56:51.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:50 smithi189 bash[38071]: debug 2024-03-07T03:56:50.982+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:06.237966+0000 front 2024-03-07T03:56:06.237663+0000 (oldest deadline 2024-03-07T03:56:32.137590+0000) 2024-03-07T03:56:51.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:50 smithi189 bash[38071]: debug 2024-03-07T03:56:50.982+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.739978+0000 front 2024-03-07T03:56:16.739736+0000 (oldest deadline 2024-03-07T03:56:42.639475+0000) 2024-03-07T03:56:51.342 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:50 smithi189 bash[38071]: debug 2024-03-07T03:56:50.982+0000 7f4fef54d700 -1 osd.6 1101 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-03-07T03:56:24.941082+0000 front 2024-03-07T03:56:24.941293+0000 (oldest deadline 2024-03-07T03:56:47.840929+0000) 2024-03-07T03:56:51.725 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:51 smithi189 bash[43534]: debug 2024-03-07T03:56:51.450+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:51.726 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:51 smithi189 bash[43534]: debug 2024-03-07T03:56:51.450+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:51.726 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:51 smithi189 bash[43534]: debug 2024-03-07T03:56:51.450+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:52.528 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:52 smithi189 bash[91106]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-6 2024-03-07T03:56:52.529 INFO:journalctl@ceph.osd.6.smithi189.stdout:Mar 07 03:56:52 smithi189 bash[91169]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-6 2024-03-07T03:56:52.529 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:52 smithi189 bash[43534]: debug 2024-03-07T03:56:52.442+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:52.529 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:52 smithi189 bash[43534]: debug 2024-03-07T03:56:52.442+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:52.529 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:52 smithi189 bash[43534]: debug 2024-03-07T03:56:52.442+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:52.529 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:52 smithi189 bash[43534]: debug 2024-03-07T03:56:52.442+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-03-07T03:56:20.614996+0000 front 2024-03-07T03:56:20.615043+0000 (oldest deadline 2024-03-07T03:56:52.415494+0000) 2024-03-07T03:56:53.591 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:53 smithi189 bash[43534]: debug 2024-03-07T03:56:53.418+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:53.591 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:53 smithi189 bash[43534]: debug 2024-03-07T03:56:53.418+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:53.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:53 smithi189 bash[43534]: debug 2024-03-07T03:56:53.418+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:53.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:53 smithi189 bash[43534]: debug 2024-03-07T03:56:53.418+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-03-07T03:56:20.614996+0000 front 2024-03-07T03:56:20.615043+0000 (oldest deadline 2024-03-07T03:56:52.415494+0000) 2024-03-07T03:56:53.820 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:53 smithi145 bash[57902]: ts=2024-03-07T03:56:53.519Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:53.821 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:53 smithi145 bash[57902]: ts=2024-03-07T03:56:53.520Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:56:53.821 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:56:53 smithi145 bash[57902]: ts=2024-03-07T03:56:53.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:56:54.353 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.6.service' 2024-03-07T03:56:54.382 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:56:54.383 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-03-07T03:56:54.383 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-03-07T03:56:54.383 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.7 2024-03-07T03:56:54.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:54 smithi189 bash[43534]: debug 2024-03-07T03:56:54.414+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:54.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:54 smithi189 bash[43534]: debug 2024-03-07T03:56:54.418+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:54.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:54 smithi189 bash[43534]: debug 2024-03-07T03:56:54.418+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:54.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:54 smithi189 bash[43534]: debug 2024-03-07T03:56:54.418+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-03-07T03:56:20.614996+0000 front 2024-03-07T03:56:20.615043+0000 (oldest deadline 2024-03-07T03:56:52.415494+0000) 2024-03-07T03:56:54.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:54 smithi189 systemd[1]: Stopping Ceph osd.7 for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:56:54.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:54 smithi189 bash[43534]: debug 2024-03-07T03:56:54.546+0000 7fa239cbc700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:56:54.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:54 smithi189 bash[43534]: debug 2024-03-07T03:56:54.546+0000 7fa239cbc700 -1 osd.7 1101 *** Got signal Terminated *** 2024-03-07T03:56:54.843 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:54 smithi189 bash[43534]: debug 2024-03-07T03:56:54.546+0000 7fa239cbc700 -1 osd.7 1101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:56:55.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:55 smithi189 bash[43534]: debug 2024-03-07T03:56:55.390+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:55.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:55 smithi189 bash[43534]: debug 2024-03-07T03:56:55.390+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:55.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:55 smithi189 bash[43534]: debug 2024-03-07T03:56:55.390+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:55.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:55 smithi189 bash[43534]: debug 2024-03-07T03:56:55.390+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-03-07T03:56:20.614996+0000 front 2024-03-07T03:56:20.615043+0000 (oldest deadline 2024-03-07T03:56:52.415494+0000) 2024-03-07T03:56:56.841 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:56 smithi189 bash[43534]: debug 2024-03-07T03:56:56.354+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:56.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:56 smithi189 bash[43534]: debug 2024-03-07T03:56:56.354+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:56.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:56 smithi189 bash[43534]: debug 2024-03-07T03:56:56.354+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:56.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:56 smithi189 bash[43534]: debug 2024-03-07T03:56:56.354+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-03-07T03:56:20.614996+0000 front 2024-03-07T03:56:20.615043+0000 (oldest deadline 2024-03-07T03:56:52.415494+0000) 2024-03-07T03:56:56.842 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:56 smithi189 bash[43534]: debug 2024-03-07T03:56:56.354+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.189:6804 osd.4 since back 2024-03-07T03:56:32.416282+0000 front 2024-03-07T03:56:32.416429+0000 (oldest deadline 2024-03-07T03:56:55.916111+0000) 2024-03-07T03:56:57.591 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:57 smithi189 bash[43534]: debug 2024-03-07T03:56:57.322+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:57.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:57 smithi189 bash[43534]: debug 2024-03-07T03:56:57.322+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:57.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:57 smithi189 bash[43534]: debug 2024-03-07T03:56:57.322+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:57.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:57 smithi189 bash[43534]: debug 2024-03-07T03:56:57.322+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-03-07T03:56:20.614996+0000 front 2024-03-07T03:56:20.615043+0000 (oldest deadline 2024-03-07T03:56:52.415494+0000) 2024-03-07T03:56:57.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:57 smithi189 bash[43534]: debug 2024-03-07T03:56:57.322+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.189:6804 osd.4 since back 2024-03-07T03:56:32.416282+0000 front 2024-03-07T03:56:32.416429+0000 (oldest deadline 2024-03-07T03:56:55.916111+0000) 2024-03-07T03:56:58.591 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:58 smithi189 bash[43534]: debug 2024-03-07T03:56:58.314+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:58.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:58 smithi189 bash[43534]: debug 2024-03-07T03:56:58.314+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:58.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:58 smithi189 bash[43534]: debug 2024-03-07T03:56:58.314+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:58.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:58 smithi189 bash[43534]: debug 2024-03-07T03:56:58.314+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-03-07T03:56:20.614996+0000 front 2024-03-07T03:56:20.615043+0000 (oldest deadline 2024-03-07T03:56:52.415494+0000) 2024-03-07T03:56:58.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:58 smithi189 bash[43534]: debug 2024-03-07T03:56:58.314+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.189:6804 osd.4 since back 2024-03-07T03:56:32.416282+0000 front 2024-03-07T03:56:32.416429+0000 (oldest deadline 2024-03-07T03:56:55.916111+0000) 2024-03-07T03:56:59.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:59 smithi189 bash[43534]: debug 2024-03-07T03:56:59.270+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6806 osd.0 since back 2024-03-07T03:55:56.809409+0000 front 2024-03-07T03:55:56.809386+0000 (oldest deadline 2024-03-07T03:56:21.509174+0000) 2024-03-07T03:56:59.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:59 smithi189 bash[43534]: debug 2024-03-07T03:56:59.270+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6814 osd.1 since back 2024-03-07T03:56:07.212317+0000 front 2024-03-07T03:56:07.212358+0000 (oldest deadline 2024-03-07T03:56:30.111988+0000) 2024-03-07T03:56:59.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:59 smithi189 bash[43534]: debug 2024-03-07T03:56:59.270+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6822 osd.2 since back 2024-03-07T03:56:16.013526+0000 front 2024-03-07T03:56:16.013760+0000 (oldest deadline 2024-03-07T03:56:38.913439+0000) 2024-03-07T03:56:59.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:59 smithi189 bash[43534]: debug 2024-03-07T03:56:59.270+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.145:6830 osd.3 since back 2024-03-07T03:56:20.614996+0000 front 2024-03-07T03:56:20.615043+0000 (oldest deadline 2024-03-07T03:56:52.415494+0000) 2024-03-07T03:56:59.592 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:56:59 smithi189 bash[43534]: debug 2024-03-07T03:56:59.270+0000 7fa232aab700 -1 osd.7 1101 heartbeat_check: no reply from 172.21.15.189:6804 osd.4 since back 2024-03-07T03:56:32.416282+0000 front 2024-03-07T03:56:32.416429+0000 (oldest deadline 2024-03-07T03:56:55.916111+0000) 2024-03-07T03:57:00.570 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:00 smithi145 bash[57902]: ts=2024-03-07T03:57:00.287Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:57:00.571 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:00 smithi145 bash[57902]: ts=2024-03-07T03:57:00.288Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:57:00.571 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:00 smithi145 bash[57902]: ts=2024-03-07T03:57:00.290Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:57:00.824 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:57:00 smithi189 bash[91303]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-7 2024-03-07T03:57:00.824 INFO:journalctl@ceph.osd.7.smithi189.stdout:Mar 07 03:57:00 smithi189 bash[91364]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-osd-7 2024-03-07T03:57:02.595 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@osd.7.service' 2024-03-07T03:57:02.622 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:57:02.622 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-03-07T03:57:02.623 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-03-07T03:57:02.623 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@prometheus.a 2024-03-07T03:57:02.841 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:57:02 smithi189 systemd[1]: Stopping Ceph prometheus.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:02.841 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:57:02 smithi189 bash[59814]: ts=2024-03-07T03:57:02.806Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-07T03:57:02.841 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:57:02 smithi189 bash[59814]: ts=2024-03-07T03:57:02.806Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-07T03:57:02.842 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:57:02 smithi189 bash[59814]: ts=2024-03-07T03:57:02.806Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-07T03:57:02.842 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:57:02 smithi189 bash[59814]: ts=2024-03-07T03:57:02.806Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-07T03:57:02.842 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:57:02 smithi189 bash[59814]: ts=2024-03-07T03:57:02.806Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-07T03:57:02.842 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:57:02 smithi189 bash[59814]: ts=2024-03-07T03:57:02.806Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-07T03:57:02.842 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:57:02 smithi189 bash[59814]: ts=2024-03-07T03:57:02.806Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-07T03:57:02.842 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:57:02 smithi189 bash[59814]: ts=2024-03-07T03:57:02.806Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-07T03:57:02.842 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:57:02 smithi189 bash[59814]: ts=2024-03-07T03:57:02.807Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-07T03:57:02.842 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:57:02 smithi189 bash[59814]: ts=2024-03-07T03:57:02.808Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-07T03:57:02.843 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:57:02 smithi189 bash[59814]: ts=2024-03-07T03:57:02.808Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-07T03:57:02.843 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Mar 07 03:57:02 smithi189 bash[59814]: ts=2024-03-07T03:57:02.808Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-07T03:57:03.517 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@prometheus.a.service' 2024-03-07T03:57:03.540 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:57:03.541 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-03-07T03:57:03.541 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 --force --keep-logs 2024-03-07T03:57:03.665 INFO:teuthology.orchestra.run.smithi145.stdout:Deleting cluster with fsid: 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:57:03.821 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:03 smithi145 bash[57902]: ts=2024-03-07T03:57:03.519Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:57:03.821 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:03 smithi145 bash[57902]: ts=2024-03-07T03:57:03.520Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:57:03.821 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:03 smithi145 bash[57902]: ts=2024-03-07T03:57:03.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:57:08.570 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:08 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:08.571 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:08 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:08.863 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:08 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:08.864 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:08 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:08.864 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:08 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:08.864 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:08 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:09.130 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:09 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:09.131 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:09 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:09.447 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:09 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:09.447 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:09 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:09.730 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:09 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:09.730 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:09 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:09.731 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:09 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:09.731 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:09 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:10.070 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:09 smithi145 systemd[1]: Stopping Ceph node-exporter.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:10.420 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:10 smithi145 bash[57902]: ts=2024-03-07T03:57:10.288Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:57:10.421 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:10 smithi145 bash[57902]: ts=2024-03-07T03:57:10.289Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi145.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.145:8443: connect: connection refused" 2024-03-07T03:57:10.421 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:10 smithi145 bash[57902]: ts=2024-03-07T03:57:10.290Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi189.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-03-07T03:57:10.680 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:10 smithi145 bash[178245]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-node-exporter-a 2024-03-07T03:57:10.680 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:10 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-03-07T03:57:10.680 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:10 smithi145 bash[178303]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-node-exporter-a 2024-03-07T03:57:10.680 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:10 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@node-exporter.a.service: Failed with result 'exit-code'. 2024-03-07T03:57:10.680 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:10 smithi145 systemd[1]: Stopped Ceph node-exporter.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:57:11.035 INFO:journalctl@ceph.node-exporter.a.smithi145.stdout:Mar 07 03:57:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:11.036 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:10 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:11.036 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:10 smithi145 systemd[1]: Stopping Ceph alertmanager.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:11.036 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:11 smithi145 bash[57902]: ts=2024-03-07T03:57:11.036Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-03-07T03:57:12.004 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:11 smithi145 bash[178365]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-alertmanager-a 2024-03-07T03:57:12.005 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:11 smithi145 bash[178424]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-alertmanager-a 2024-03-07T03:57:12.005 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:11 smithi145 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-03-07T03:57:12.005 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:11 smithi145 systemd[1]: Stopped Ceph alertmanager.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:57:12.320 INFO:journalctl@ceph.alertmanager.a.smithi145.stdout:Mar 07 03:57:12 smithi145 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:14.989 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 --force --keep-logs 2024-03-07T03:57:15.113 INFO:teuthology.orchestra.run.smithi189.stdout:Deleting cluster with fsid: 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:57:20.047 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:19 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:20.047 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:19 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:20.303 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:20 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:20.303 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:20 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:20.304 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:20 smithi189 bash[49584]: logger=cleanup t=2024-03-07T03:57:20.290423118Z level=info msg="Completed cleanup jobs" duration=141.500087ms 2024-03-07T03:57:20.591 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:20 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:20.591 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:20 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:20.592 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:20 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:20.592 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:20 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:20.861 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:20 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:20.862 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:20 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:21.200 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:20 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:21.201 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:20 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:21.591 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:21 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:21.592 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:21 smithi189 systemd[1]: Stopping Ceph node-exporter.b for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:21.592 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:21 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:22.539 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:22 smithi189 bash[91825]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-node-exporter-b 2024-03-07T03:57:22.539 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:22 smithi189 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-03-07T03:57:22.539 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:22 smithi189 bash[91886]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-node-exporter-b 2024-03-07T03:57:22.539 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:22 smithi189 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@node-exporter.b.service: Failed with result 'exit-code'. 2024-03-07T03:57:22.540 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:22 smithi189 systemd[1]: Stopped Ceph node-exporter.b for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:57:22.797 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:22 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:22.797 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:22 smithi189 systemd[1]: Stopping Ceph grafana.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:22.797 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Mar 07 03:57:22 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:23.534 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:23 smithi189 bash[91949]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-grafana-a 2024-03-07T03:57:23.534 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:23 smithi189 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-03-07T03:57:23.534 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:23 smithi189 bash[92008]: Error response from daemon: No such container: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715-grafana-a 2024-03-07T03:57:23.534 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:23 smithi189 systemd[1]: ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-07T03:57:23.535 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:23 smithi189 systemd[1]: Stopped Ceph grafana.a for 8947a0f8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:57:23.841 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Mar 07 03:57:23 smithi189 systemd[1]: /etc/systemd/system/ceph-8947a0f8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:57:32.010 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-07T03:57:32.024 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-07T03:57:32.036 INFO:tasks.cephadm:Archiving crash dumps... 2024-03-07T03:57:32.038 DEBUG:teuthology.misc:Transferring archived files from smithi145:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/crash to /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583725/remote/smithi145/crash 2024-03-07T03:57:32.039 DEBUG:teuthology.orchestra.run.smithi145:> sudo tar c -f - -C /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/crash -- . 2024-03-07T03:57:32.084 INFO:teuthology.orchestra.run.smithi145.stderr:tar: /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/crash: Cannot open: No such file or directory 2024-03-07T03:57:32.084 INFO:teuthology.orchestra.run.smithi145.stderr:tar: Error is not recoverable: exiting now 2024-03-07T03:57:32.086 DEBUG:teuthology.misc:Transferring archived files from smithi189:/var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/crash to /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583725/remote/smithi189/crash 2024-03-07T03:57:32.087 DEBUG:teuthology.orchestra.run.smithi189:> sudo tar c -f - -C /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/crash -- . 2024-03-07T03:57:32.098 INFO:teuthology.orchestra.run.smithi189.stderr:tar: /var/lib/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/crash: Cannot open: No such file or directory 2024-03-07T03:57:32.099 INFO:teuthology.orchestra.run.smithi189.stderr:tar: Error is not recoverable: exiting now 2024-03-07T03:57:32.100 INFO:tasks.cephadm:Checking cluster log for badness... 2024-03-07T03:57:32.100 DEBUG:teuthology.orchestra.run.smithi145:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-07T03:57:32.141 INFO:teuthology.orchestra.run.smithi145.stdout:2024-03-07T02:58:04.741558+0000 mon.a (mon.0) 181 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-03-07T03:57:32.142 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-03-07T03:57:32.142 INFO:tasks.cephadm:Compressing logs... 2024-03-07T03:57:32.142 DEBUG:teuthology.orchestra.run.smithi145:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-07T03:57:32.186 DEBUG:teuthology.orchestra.run.smithi189:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-07T03:57:32.195 INFO:teuthology.orchestra.run.smithi145.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-03-07T03:57:32.198 INFO:teuthology.orchestra.run.smithi189.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-03-07T03:57:32.203 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-07T03:57:32.203 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mgr.y.log 2024-03-07T03:57:32.204 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.0.log 2024-03-07T03:57:32.204 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.2.log 2024-03-07T03:57:32.208 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mon.a.log 2024-03-07T03:57:32.208 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.0.log: /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.cephadm.log 2024-03-07T03:57:32.208 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.3.log 2024-03-07T03:57:32.208 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mon.a.log: /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.1.log 2024-03-07T03:57:32.208 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mds.foo.smithi145.zmgvgw.log 2024-03-07T03:57:32.208 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.3.log: 90.7% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.cephadm.log.gz 2024-03-07T03:57:32.209 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.log 2024-03-07T03:57:32.209 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mds.foo.smithi145.zmgvgw.log: 87.0% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mds.foo.smithi145.zmgvgw.log.gz 2024-03-07T03:57:32.209 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-volume.log 2024-03-07T03:57:32.211 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-07T03:57:32.212 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.7.log 2024-03-07T03:57:32.212 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.6.log 2024-03-07T03:57:32.213 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.log: 92.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-07T03:57:32.214 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mon.c.log 2024-03-07T03:57:32.216 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.7.log: /var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.cephadm.log 2024-03-07T03:57:32.216 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.5.log 2024-03-07T03:57:32.216 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mgr.x.log 2024-03-07T03:57:32.216 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.log 2024-03-07T03:57:32.216 INFO:teuthology.orchestra.run.smithi189.stderr: 83.9% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.cephadm.log.gz 2024-03-07T03:57:32.216 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-volume.log 2024-03-07T03:57:32.216 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.log: gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.audit.log 2024-03-07T03:57:32.217 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mds.foo.smithi189.zkinbe.log 2024-03-07T03:57:32.222 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.audit.log: 90.6% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.audit.log.gz 2024-03-07T03:57:32.222 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mon.b.log 2024-03-07T03:57:32.223 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mds.foo.smithi189.zkinbe.log: 85.4% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mds.foo.smithi189.zkinbe.log.gz 2024-03-07T03:57:32.223 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.4.log 2024-03-07T03:57:32.224 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mon.b.log: 84.2% -- replaced with /var/log/ceph/cephadm.log.gz 86.4% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.log.gz 2024-03-07T03:57:32.224 INFO:teuthology.orchestra.run.smithi189.stderr: 2024-03-07T03:57:32.225 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-volume.log: 92.4% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.log.gz 2024-03-07T03:57:32.225 INFO:teuthology.orchestra.run.smithi145.stderr:gzip -5 --verbose -- /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.audit.log 2024-03-07T03:57:32.233 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.4.log: 91.3% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-volume.log.gz 2024-03-07T03:57:32.238 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mon.c.log: 91.1% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-volume.log.gz 2024-03-07T03:57:32.249 INFO:teuthology.orchestra.run.smithi145.stderr:/var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.audit.log: 94.3% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph.audit.log.gz 2024-03-07T03:57:32.275 INFO:teuthology.orchestra.run.smithi189.stderr: 89.9% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mgr.x.log.gz 2024-03-07T03:57:32.887 INFO:teuthology.orchestra.run.smithi145.stderr: 89.5% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mgr.y.log.gz 2024-03-07T03:57:33.273 INFO:teuthology.orchestra.run.smithi145.stderr: 92.5% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mon.c.log.gz 2024-03-07T03:57:33.359 INFO:teuthology.orchestra.run.smithi189.stderr: 92.6% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mon.b.log.gz 2024-03-07T03:57:34.471 INFO:teuthology.orchestra.run.smithi145.stderr: 90.8% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-mon.a.log.gz 2024-03-07T03:57:39.927 INFO:teuthology.orchestra.run.smithi189.stderr: 94.3% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.5.log.gz 2024-03-07T03:57:41.923 INFO:teuthology.orchestra.run.smithi145.stderr: 94.5% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.2.log.gz 2024-03-07T03:57:41.952 INFO:teuthology.orchestra.run.smithi145.stderr: 94.4% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.0.log.gz 2024-03-07T03:57:42.136 INFO:teuthology.orchestra.run.smithi189.stderr: 94.9% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.7.log.gz 2024-03-07T03:57:42.146 INFO:teuthology.orchestra.run.smithi145.stderr: 94.4% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.1.log.gz 2024-03-07T03:57:42.654 INFO:teuthology.orchestra.run.smithi145.stderr: 94.4% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.3.log.gz 2024-03-07T03:57:42.656 INFO:teuthology.orchestra.run.smithi145.stderr: 2024-03-07T03:57:42.656 INFO:teuthology.orchestra.run.smithi145.stderr:real 0m10.468s 2024-03-07T03:57:42.656 INFO:teuthology.orchestra.run.smithi145.stderr:user 0m42.305s 2024-03-07T03:57:42.656 INFO:teuthology.orchestra.run.smithi145.stderr:sys 0m1.634s 2024-03-07T03:57:42.802 INFO:teuthology.orchestra.run.smithi189.stderr: 94.7% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.4.log.gz 2024-03-07T03:57:42.961 INFO:teuthology.orchestra.run.smithi189.stderr: 94.9% -- replaced with /var/log/ceph/8947a0f8-dc2d-11ee-95c6-87774f69a715/ceph-osd.6.log.gz 2024-03-07T03:57:42.963 INFO:teuthology.orchestra.run.smithi189.stderr: 2024-03-07T03:57:42.963 INFO:teuthology.orchestra.run.smithi189.stderr:real 0m10.773s 2024-03-07T03:57:42.963 INFO:teuthology.orchestra.run.smithi189.stderr:user 0m38.661s 2024-03-07T03:57:42.964 INFO:teuthology.orchestra.run.smithi189.stderr:sys 0m1.570s 2024-03-07T03:57:42.964 INFO:tasks.cephadm:Archiving logs... 2024-03-07T03:57:42.964 DEBUG:teuthology.misc:Transferring archived files from smithi145:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583725/remote/smithi145/log 2024-03-07T03:57:42.965 DEBUG:teuthology.orchestra.run.smithi145:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-07T03:57:47.899 DEBUG:teuthology.misc:Transferring archived files from smithi189:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583725/remote/smithi189/log 2024-03-07T03:57:47.900 DEBUG:teuthology.orchestra.run.smithi189:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-07T03:57:52.408 INFO:tasks.cephadm:Removing cluster... 2024-03-07T03:57:52.409 DEBUG:teuthology.orchestra.run.smithi145:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 --force 2024-03-07T03:57:52.532 INFO:teuthology.orchestra.run.smithi145.stdout:Deleting cluster with fsid: 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:57:53.933 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 8947a0f8-dc2d-11ee-95c6-87774f69a715 --force 2024-03-07T03:57:54.058 INFO:teuthology.orchestra.run.smithi189.stdout:Deleting cluster with fsid: 8947a0f8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:57:55.453 INFO:tasks.cephadm:Removing cephadm ... 2024-03-07T03:57:55.453 DEBUG:teuthology.orchestra.run.smithi145:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-07T03:57:55.459 DEBUG:teuthology.orchestra.run.smithi189:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-07T03:57:55.464 INFO:tasks.cephadm:Teardown complete 2024-03-07T03:57:55.465 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-03-07T03:57:55.476 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-03-07T03:57:55.476 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-07T03:57:55.502 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-07T03:57:55.569 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-03-07T03:57:55.579 INFO:teuthology.task.clock:Checking final clock skew... 2024-03-07T03:57:55.579 DEBUG:teuthology.orchestra.run.smithi145:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-07T03:57:55.582 DEBUG:teuthology.orchestra.run.smithi189:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-07T03:57:55.596 INFO:teuthology.orchestra.run.smithi145.stdout: remote refid st t when poll reach delay offset jitter 2024-03-07T03:57:55.596 INFO:teuthology.orchestra.run.smithi145.stdout:============================================================================== 2024-03-07T03:57:55.597 INFO:teuthology.orchestra.run.smithi145.stdout:*hv01.front.sepi 44.4.53.2 2 u 56 128 377 0.097 -4.165 0.343 2024-03-07T03:57:55.597 INFO:teuthology.orchestra.run.smithi145.stdout:+hv02.front.sepi 63.231.80.2 3 u 64 128 377 0.095 +0.989 0.310 2024-03-07T03:57:55.597 INFO:teuthology.orchestra.run.smithi145.stdout:+hv03.front.sepi 74.6.168.72 3 u 62 128 377 0.094 -0.745 0.313 2024-03-07T03:57:55.597 INFO:teuthology.orchestra.run.smithi145.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-03-07T03:57:55.598 INFO:teuthology.orchestra.run.smithi189.stdout: remote refid st t when poll reach delay offset jitter 2024-03-07T03:57:55.598 INFO:teuthology.orchestra.run.smithi189.stdout:============================================================================== 2024-03-07T03:57:55.599 INFO:teuthology.orchestra.run.smithi189.stdout:*hv01.front.sepi 44.4.53.2 2 u 49 128 377 0.233 -4.094 0.087 2024-03-07T03:57:55.599 INFO:teuthology.orchestra.run.smithi189.stdout:+hv02.front.sepi 63.231.80.2 3 u 65 128 377 0.211 +0.934 0.128 2024-03-07T03:57:55.599 INFO:teuthology.orchestra.run.smithi189.stdout:+hv03.front.sepi 74.6.168.72 3 u 115 128 377 0.154 -0.734 0.082 2024-03-07T03:57:55.599 INFO:teuthology.orchestra.run.smithi189.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-03-07T03:57:55.600 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-03-07T03:57:55.610 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-03-07T03:57:55.610 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-03-07T03:57:55.634 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-03-07T03:57:55.667 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-03-07T03:57:55.701 INFO:teuthology.task.internal:Duration was 4674.610238 seconds 2024-03-07T03:57:55.702 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-03-07T03:57:55.737 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-03-07T03:57:55.738 DEBUG:teuthology.orchestra.run.smithi145:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-07T03:57:55.740 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-07T03:57:55.776 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-03-07T03:57:55.776 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi145.front.sepia.ceph.com 2024-03-07T03:57:55.776 DEBUG:teuthology.orchestra.run.smithi145:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-07T03:57:55.830 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi189.front.sepia.ceph.com 2024-03-07T03:57:55.830 DEBUG:teuthology.orchestra.run.smithi189:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-07T03:57:55.840 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-03-07T03:57:55.841 DEBUG:teuthology.orchestra.run.smithi145:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-07T03:57:55.874 DEBUG:teuthology.orchestra.run.smithi189:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-07T03:57:55.949 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-03-07T03:57:55.950 DEBUG:teuthology.orchestra.run.smithi145:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-07T03:57:55.953 DEBUG:teuthology.orchestra.run.smithi189:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-07T03:57:56.167 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-03-07T03:57:56.179 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-03-07T03:57:56.179 DEBUG:teuthology.orchestra.run.smithi145:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-07T03:57:56.193 DEBUG:teuthology.orchestra.run.smithi189:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-07T03:57:56.206 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-03-07T03:57:56.218 DEBUG:teuthology.orchestra.run.smithi145:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-07T03:57:56.238 DEBUG:teuthology.orchestra.run.smithi189:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-07T03:57:56.245 INFO:teuthology.orchestra.run.smithi145.stdout:kernel.core_pattern = core 2024-03-07T03:57:56.259 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern = core 2024-03-07T03:57:56.278 DEBUG:teuthology.orchestra.run.smithi145:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-07T03:57:56.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-07T03:57:56.308 DEBUG:teuthology.orchestra.run.smithi189:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-07T03:57:56.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-07T03:57:56.327 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-03-07T03:57:56.338 INFO:teuthology.task.internal:Transferring archived files... 2024-03-07T03:57:56.339 DEBUG:teuthology.misc:Transferring archived files from smithi145:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583725/remote/smithi145 2024-03-07T03:57:56.339 DEBUG:teuthology.orchestra.run.smithi145:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-07T03:57:56.382 DEBUG:teuthology.misc:Transferring archived files from smithi189:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583725/remote/smithi189 2024-03-07T03:57:56.382 DEBUG:teuthology.orchestra.run.smithi189:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-07T03:57:56.410 INFO:teuthology.task.internal:Removing archive directory... 2024-03-07T03:57:56.410 DEBUG:teuthology.orchestra.run.smithi145:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-07T03:57:56.418 DEBUG:teuthology.orchestra.run.smithi189:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-07T03:57:56.453 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-03-07T03:57:56.466 INFO:teuthology.task.internal:Not uploading archives. 2024-03-07T03:57:56.466 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-03-07T03:57:56.494 INFO:teuthology.task.internal:Tidying up after the test... 2024-03-07T03:57:56.495 DEBUG:teuthology.orchestra.run.smithi145:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-07T03:57:56.497 DEBUG:teuthology.orchestra.run.smithi189:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-07T03:57:56.500 INFO:teuthology.orchestra.run.smithi145.stdout: 658466 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 7 03:57 /home/ubuntu/cephtest 2024-03-07T03:57:56.503 INFO:teuthology.orchestra.run.smithi189.stdout: 658633 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 7 03:57 /home/ubuntu/cephtest 2024-03-07T03:57:56.506 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-03-07T03:57:56.527 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-03-07T03:57:56.595 INFO:teuthology.nuke:Checking targets against current locks 2024-03-07T03:57:56.621 DEBUG:teuthology.nuke:shortname: smithi145 2024-03-07T03:57:56.621 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-07T03:57:56.642 DEBUG:teuthology.nuke:shortname: smithi189 2024-03-07T03:57:56.643 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-07T03:57:56.646 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi145.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583725', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-07 02:26:06.344561', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMAL6Vof3GB7HXmCHAMjxiFOSDAwUGkVVUw2QFv3Yrl6zMKqu7qDxT/v4ir4M0Zv9hvHfDpa/xMU3HykgvyQNfU='} 2024-03-07T03:57:56.670 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi189.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583725', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-07 02:26:06.343249', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB5LvkjG6SOg20+l0hQzF1QR+7a+iTehPmeeXt6RZI2ajhMOpI1dHaL3E8yUeZ6sEIwh31Aqph0XFwy+7W5Di/w='} 2024-03-07T03:57:56.695 INFO:teuthology.orchestra.console.smithi145:Power off 2024-03-07T03:57:56.695 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-07T03:57:56.720 DEBUG:teuthology.orchestra.console.smithi145:power off output: Chassis Power Control: Down/Off 2024-03-07T03:57:56.721 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:57:56.745 INFO:teuthology.orchestra.console.smithi189:Power off 2024-03-07T03:57:56.746 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-07T03:57:56.756 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is on 2024-03-07T03:57:56.771 DEBUG:teuthology.orchestra.console.smithi189:power off output: Chassis Power Control: Down/Off 2024-03-07T03:57:56.771 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:57:56.795 DEBUG:teuthology.orchestra.console.smithi189:check power output: Chassis Power is on 2024-03-07T03:58:00.758 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:58:00.796 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:58:00.871 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is on 2024-03-07T03:58:00.909 DEBUG:teuthology.orchestra.console.smithi189:check power output: Chassis Power is on 2024-03-07T03:58:04.876 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:58:04.910 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:58:04.988 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is on 2024-03-07T03:58:05.020 DEBUG:teuthology.orchestra.console.smithi189:check power output: Chassis Power is on 2024-03-07T03:58:08.990 DEBUG:teuthology.orchestra.console.smithi145:pexpect command: ipmitool -H smithi145.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:58:09.022 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:58:09.100 DEBUG:teuthology.orchestra.console.smithi145:check power output: Chassis Power is off 2024-03-07T03:58:09.132 DEBUG:teuthology.orchestra.console.smithi189:check power output: Chassis Power is off 2024-03-07T03:58:09.201 INFO:teuthology.orchestra.console.smithi145:Power off completed 2024-03-07T03:58:09.233 INFO:teuthology.orchestra.console.smithi189:Power off completed 2024-03-07T03:58:09.338 INFO:teuthology.run:Summary data: description: upgrade:reef-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 4674.610237836838 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=2653ddcde20c4add9c65b519e9b7ccd6 status: fail success: false 2024-03-07T03:58:09.338 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-07T03:58:09.416 INFO:teuthology.run:FAIL